blob: d5840381804e62cce90cc35a36c1e9340f685bea [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Tao Bao89fbb0f2017-01-10 10:47:58 -080015from __future__ import print_function
16
Doug Zongkerea5d7a92010-09-12 15:26:16 -070017import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070018import errno
Doug Zongkereef39442009-04-02 12:14:19 -070019import getopt
20import getpass
Narayan Kamatha07bf042017-08-14 14:49:21 +010021import gzip
Doug Zongker05d3dea2009-06-22 11:32:31 -070022import imp
Doug Zongkereef39442009-04-02 12:14:19 -070023import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080024import platform
Doug Zongkereef39442009-04-02 12:14:19 -070025import re
T.R. Fullhart37e10522013-03-18 10:31:26 -070026import shlex
Doug Zongkereef39442009-04-02 12:14:19 -070027import shutil
Tao Baoc765cca2018-01-31 17:32:40 -080028import string
Doug Zongkereef39442009-04-02 12:14:19 -070029import subprocess
30import sys
31import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070032import threading
33import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070034import zipfile
Tao Bao12d87fc2018-01-31 12:18:52 -080035from hashlib import sha1, sha256
Doug Zongkereef39442009-04-02 12:14:19 -070036
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070037import blockimgdiff
Tao Baoc765cca2018-01-31 17:32:40 -080038import sparse_img
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070039
Dan Albert8b72aef2015-03-23 19:13:21 -070040class Options(object):
41 def __init__(self):
42 platform_search_path = {
43 "linux2": "out/host/linux-x86",
44 "darwin": "out/host/darwin-x86",
Doug Zongker85448772014-09-09 14:59:20 -070045 }
Doug Zongker85448772014-09-09 14:59:20 -070046
Dan Albert8b72aef2015-03-23 19:13:21 -070047 self.search_path = platform_search_path.get(sys.platform, None)
48 self.signapk_path = "framework/signapk.jar" # Relative to search_path
Alex Klyubin9667b182015-12-10 13:38:50 -080049 self.signapk_shared_library_path = "lib64" # Relative to search_path
Dan Albert8b72aef2015-03-23 19:13:21 -070050 self.extra_signapk_args = []
51 self.java_path = "java" # Use the one on the path by default.
Tao Baoe95540e2016-11-08 12:08:53 -080052 self.java_args = ["-Xmx2048m"] # The default JVM args.
Dan Albert8b72aef2015-03-23 19:13:21 -070053 self.public_key_suffix = ".x509.pem"
54 self.private_key_suffix = ".pk8"
Dan Albertcd9ecc02015-03-27 16:37:23 -070055 # use otatools built boot_signer by default
56 self.boot_signer_path = "boot_signer"
Baligh Uddin601ddea2015-06-09 15:48:14 -070057 self.boot_signer_args = []
58 self.verity_signer_path = None
59 self.verity_signer_args = []
Dan Albert8b72aef2015-03-23 19:13:21 -070060 self.verbose = False
61 self.tempfiles = []
62 self.device_specific = None
63 self.extras = {}
64 self.info_dict = None
Tao Bao6f0b2192015-10-13 16:37:12 -070065 self.source_info_dict = None
66 self.target_info_dict = None
Dan Albert8b72aef2015-03-23 19:13:21 -070067 self.worker_threads = None
Tao Bao575d68a2015-08-07 19:49:45 -070068 # Stash size cannot exceed cache_size * threshold.
69 self.cache_size = None
70 self.stash_threshold = 0.8
Dan Albert8b72aef2015-03-23 19:13:21 -070071
72
73OPTIONS = Options()
Doug Zongkereef39442009-04-02 12:14:19 -070074
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080075
76# Values for "certificate" in apkcerts that mean special things.
77SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
78
Tao Bao9dd909e2017-11-14 11:27:32 -080079
80# The partitions allowed to be signed by AVB (Android verified boot 2.0).
Jaekyun Seokb7735d82017-11-27 17:04:47 +090081AVB_PARTITIONS = ('boot', 'recovery', 'system', 'vendor', 'product', 'dtbo')
Tao Bao9dd909e2017-11-14 11:27:32 -080082
83
Tianjie Xu209db462016-05-24 17:34:52 -070084class ErrorCode(object):
85 """Define error_codes for failures that happen during the actual
86 update package installation.
87
88 Error codes 0-999 are reserved for failures before the package
89 installation (i.e. low battery, package verification failure).
90 Detailed code in 'bootable/recovery/error_code.h' """
91
92 SYSTEM_VERIFICATION_FAILURE = 1000
93 SYSTEM_UPDATE_FAILURE = 1001
94 SYSTEM_UNEXPECTED_CONTENTS = 1002
95 SYSTEM_NONZERO_CONTENTS = 1003
96 SYSTEM_RECOVER_FAILURE = 1004
97 VENDOR_VERIFICATION_FAILURE = 2000
98 VENDOR_UPDATE_FAILURE = 2001
99 VENDOR_UNEXPECTED_CONTENTS = 2002
100 VENDOR_NONZERO_CONTENTS = 2003
101 VENDOR_RECOVER_FAILURE = 2004
102 OEM_PROP_MISMATCH = 3000
103 FINGERPRINT_MISMATCH = 3001
104 THUMBPRINT_MISMATCH = 3002
105 OLDER_BUILD = 3003
106 DEVICE_MISMATCH = 3004
107 BAD_PATCH_FILE = 3005
108 INSUFFICIENT_CACHE_SPACE = 3006
109 TUNE_PARTITION_FAILURE = 3007
110 APPLY_PATCH_FAILURE = 3008
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800111
Dan Albert8b72aef2015-03-23 19:13:21 -0700112class ExternalError(RuntimeError):
113 pass
Doug Zongkereef39442009-04-02 12:14:19 -0700114
115
Tao Bao39451582017-05-04 11:10:47 -0700116def Run(args, verbose=None, **kwargs):
117 """Create and return a subprocess.Popen object.
118
119 Caller can specify if the command line should be printed. The global
120 OPTIONS.verbose will be used if not specified.
121 """
122 if verbose is None:
123 verbose = OPTIONS.verbose
124 if verbose:
Tao Bao89fbb0f2017-01-10 10:47:58 -0800125 print(" running: ", " ".join(args))
Doug Zongkereef39442009-04-02 12:14:19 -0700126 return subprocess.Popen(args, **kwargs)
127
128
Tao Baoc765cca2018-01-31 17:32:40 -0800129def RoundUpTo4K(value):
130 rounded_up = value + 4095
131 return rounded_up - (rounded_up % 4096)
132
133
Ying Wang7e6d4e42010-12-13 16:25:36 -0800134def CloseInheritedPipes():
135 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
136 before doing other work."""
137 if platform.system() != "Darwin":
138 return
139 for d in range(3, 1025):
140 try:
141 stat = os.fstat(d)
142 if stat is not None:
143 pipebit = stat[0] & 0x1000
144 if pipebit != 0:
145 os.close(d)
146 except OSError:
147 pass
148
149
Tao Bao2c15d9e2015-07-09 11:51:16 -0700150def LoadInfoDict(input_file, input_dir=None):
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700151 """Read and parse the META/misc_info.txt key/value pairs from the
152 input target files and return a dict."""
153
Doug Zongkerc9253822014-02-04 12:17:58 -0800154 def read_helper(fn):
Dan Albert8b72aef2015-03-23 19:13:21 -0700155 if isinstance(input_file, zipfile.ZipFile):
156 return input_file.read(fn)
Doug Zongkerc9253822014-02-04 12:17:58 -0800157 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700158 path = os.path.join(input_file, *fn.split("/"))
Doug Zongkerc9253822014-02-04 12:17:58 -0800159 try:
160 with open(path) as f:
161 return f.read()
Dan Albert8b72aef2015-03-23 19:13:21 -0700162 except IOError as e:
Doug Zongkerc9253822014-02-04 12:17:58 -0800163 if e.errno == errno.ENOENT:
164 raise KeyError(fn)
Tao Bao6cd54732017-02-27 15:12:05 -0800165
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700166 try:
Michael Runge6e836112014-04-15 17:40:21 -0700167 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n"))
Doug Zongker37974732010-09-16 17:44:38 -0700168 except KeyError:
Tao Bao6cd54732017-02-27 15:12:05 -0800169 raise ValueError("can't find META/misc_info.txt in input target-files")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700170
Tao Bao6cd54732017-02-27 15:12:05 -0800171 assert "recovery_api_version" in d
Tao Baod1de6f32017-03-01 16:38:48 -0800172 assert "fstab_version" in d
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800173
Tao Bao84e75682015-07-19 02:38:53 -0700174 # A few properties are stored as links to the files in the out/ directory.
175 # It works fine with the build system. However, they are no longer available
176 # when (re)generating from target_files zip. If input_dir is not None, we
177 # are doing repacking. Redirect those properties to the actual files in the
178 # unzipped directory.
Tao Bao2c15d9e2015-07-09 11:51:16 -0700179 if input_dir is not None:
Stephen Smalleyd3a803e2015-08-04 14:59:06 -0400180 # We carry a copy of file_contexts.bin under META/. If not available,
181 # search BOOT/RAMDISK/. Note that sometimes we may need a different file
Tao Bao84e75682015-07-19 02:38:53 -0700182 # to build images than the one running on device, such as when enabling
183 # system_root_image. In that case, we must have the one for image
184 # generation copied to META/.
Tao Bao79735a62015-08-28 10:52:03 -0700185 fc_basename = os.path.basename(d.get("selinux_fc", "file_contexts"))
186 fc_config = os.path.join(input_dir, "META", fc_basename)
Tao Bao84e75682015-07-19 02:38:53 -0700187 if d.get("system_root_image") == "true":
188 assert os.path.exists(fc_config)
Tao Bao2c15d9e2015-07-09 11:51:16 -0700189 if not os.path.exists(fc_config):
Tao Bao79735a62015-08-28 10:52:03 -0700190 fc_config = os.path.join(input_dir, "BOOT", "RAMDISK", fc_basename)
Tao Bao2c15d9e2015-07-09 11:51:16 -0700191 if not os.path.exists(fc_config):
192 fc_config = None
193
194 if fc_config:
195 d["selinux_fc"] = fc_config
196
Tao Bao80022302018-07-20 15:20:28 -0700197 # Similarly we need to redirect "root_dir" and "root_fs_config".
Tao Bao84e75682015-07-19 02:38:53 -0700198 if d.get("system_root_image") == "true":
Tao Bao80022302018-07-20 15:20:28 -0700199 d["root_dir"] = os.path.join(input_dir, "ROOT")
200 d["root_fs_config"] = os.path.join(
Tao Bao84e75682015-07-19 02:38:53 -0700201 input_dir, "META", "root_filesystem_config.txt")
202
Tao Baof54216f2016-03-29 15:12:37 -0700203 # Redirect {system,vendor}_base_fs_file.
204 if "system_base_fs_file" in d:
205 basename = os.path.basename(d["system_base_fs_file"])
206 system_base_fs_file = os.path.join(input_dir, "META", basename)
Tao Baob079b502016-05-03 08:01:19 -0700207 if os.path.exists(system_base_fs_file):
208 d["system_base_fs_file"] = system_base_fs_file
209 else:
Tao Bao89fbb0f2017-01-10 10:47:58 -0800210 print("Warning: failed to find system base fs file: %s" % (
211 system_base_fs_file,))
Tao Baob079b502016-05-03 08:01:19 -0700212 del d["system_base_fs_file"]
Tao Baof54216f2016-03-29 15:12:37 -0700213
214 if "vendor_base_fs_file" in d:
215 basename = os.path.basename(d["vendor_base_fs_file"])
216 vendor_base_fs_file = os.path.join(input_dir, "META", basename)
Tao Baob079b502016-05-03 08:01:19 -0700217 if os.path.exists(vendor_base_fs_file):
218 d["vendor_base_fs_file"] = vendor_base_fs_file
219 else:
Tao Bao89fbb0f2017-01-10 10:47:58 -0800220 print("Warning: failed to find vendor base fs file: %s" % (
221 vendor_base_fs_file,))
Tao Baob079b502016-05-03 08:01:19 -0700222 del d["vendor_base_fs_file"]
Tao Baof54216f2016-03-29 15:12:37 -0700223
Doug Zongker37974732010-09-16 17:44:38 -0700224 def makeint(key):
225 if key in d:
226 d[key] = int(d[key], 0)
227
228 makeint("recovery_api_version")
229 makeint("blocksize")
230 makeint("system_size")
Daniel Rosenbergf4eabc32014-07-10 15:42:38 -0700231 makeint("vendor_size")
Doug Zongker37974732010-09-16 17:44:38 -0700232 makeint("userdata_size")
Ying Wang9f8e8db2011-11-04 11:37:01 -0700233 makeint("cache_size")
Doug Zongker37974732010-09-16 17:44:38 -0700234 makeint("recovery_size")
235 makeint("boot_size")
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800236 makeint("fstab_version")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700237
Tianjie Xucfa86222016-03-07 16:31:19 -0800238 system_root_image = d.get("system_root_image", None) == "true"
239 if d.get("no_recovery", None) != "true":
240 recovery_fstab_path = "RECOVERY/RAMDISK/etc/recovery.fstab"
Tao Bao48550cc2015-11-19 17:05:46 -0800241 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"],
Tianjie Xucfa86222016-03-07 16:31:19 -0800242 recovery_fstab_path, system_root_image)
243 elif d.get("recovery_as_boot", None) == "true":
244 recovery_fstab_path = "BOOT/RAMDISK/etc/recovery.fstab"
245 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"],
246 recovery_fstab_path, system_root_image)
247 else:
248 d["fstab"] = None
249
Tao Baobcd1d162017-08-26 13:10:26 -0700250 d["build.prop"] = LoadBuildProp(read_helper, 'SYSTEM/build.prop')
251 d["vendor.build.prop"] = LoadBuildProp(read_helper, 'VENDOR/build.prop')
Tao Bao12d87fc2018-01-31 12:18:52 -0800252
253 # Set up the salt (based on fingerprint or thumbprint) that will be used when
254 # adding AVB footer.
255 if d.get("avb_enable") == "true":
256 fp = None
257 if "build.prop" in d:
258 build_prop = d["build.prop"]
259 if "ro.build.fingerprint" in build_prop:
260 fp = build_prop["ro.build.fingerprint"]
261 elif "ro.build.thumbprint" in build_prop:
262 fp = build_prop["ro.build.thumbprint"]
263 if fp:
264 d["avb_salt"] = sha256(fp).hexdigest()
265
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700266 return d
267
Tao Baod1de6f32017-03-01 16:38:48 -0800268
Tao Baobcd1d162017-08-26 13:10:26 -0700269def LoadBuildProp(read_helper, prop_file):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700270 try:
Tao Baobcd1d162017-08-26 13:10:26 -0700271 data = read_helper(prop_file)
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700272 except KeyError:
Tao Baobcd1d162017-08-26 13:10:26 -0700273 print("Warning: could not read %s" % (prop_file,))
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700274 data = ""
Michael Runge6e836112014-04-15 17:40:21 -0700275 return LoadDictionaryFromLines(data.split("\n"))
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700276
Tao Baod1de6f32017-03-01 16:38:48 -0800277
Michael Runge6e836112014-04-15 17:40:21 -0700278def LoadDictionaryFromLines(lines):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700279 d = {}
Michael Runge6e836112014-04-15 17:40:21 -0700280 for line in lines:
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700281 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700282 if not line or line.startswith("#"):
283 continue
Ying Wang114b46f2014-04-15 11:24:00 -0700284 if "=" in line:
285 name, value = line.split("=", 1)
286 d[name] = value
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700287 return d
288
Tao Baod1de6f32017-03-01 16:38:48 -0800289
Tianjie Xucfa86222016-03-07 16:31:19 -0800290def LoadRecoveryFSTab(read_helper, fstab_version, recovery_fstab_path,
291 system_root_image=False):
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700292 class Partition(object):
Tao Baod1de6f32017-03-01 16:38:48 -0800293 def __init__(self, mount_point, fs_type, device, length, context):
Dan Albert8b72aef2015-03-23 19:13:21 -0700294 self.mount_point = mount_point
295 self.fs_type = fs_type
296 self.device = device
297 self.length = length
Tao Bao548eb762015-06-10 12:32:41 -0700298 self.context = context
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700299
300 try:
Tianjie Xucfa86222016-03-07 16:31:19 -0800301 data = read_helper(recovery_fstab_path)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700302 except KeyError:
Tao Bao89fbb0f2017-01-10 10:47:58 -0800303 print("Warning: could not find {}".format(recovery_fstab_path))
Jeff Davidson033fbe22011-10-26 18:08:09 -0700304 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700305
Tao Baod1de6f32017-03-01 16:38:48 -0800306 assert fstab_version == 2
307
308 d = {}
309 for line in data.split("\n"):
310 line = line.strip()
311 if not line or line.startswith("#"):
312 continue
313
314 # <src> <mnt_point> <type> <mnt_flags and options> <fs_mgr_flags>
315 pieces = line.split()
316 if len(pieces) != 5:
317 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
318
319 # Ignore entries that are managed by vold.
320 options = pieces[4]
321 if "voldmanaged=" in options:
322 continue
323
324 # It's a good line, parse it.
325 length = 0
326 options = options.split(",")
327 for i in options:
328 if i.startswith("length="):
329 length = int(i[7:])
Doug Zongker086cbb02011-02-17 15:54:20 -0800330 else:
Tao Baod1de6f32017-03-01 16:38:48 -0800331 # Ignore all unknown options in the unified fstab.
Dan Albert8b72aef2015-03-23 19:13:21 -0700332 continue
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800333
Tao Baod1de6f32017-03-01 16:38:48 -0800334 mount_flags = pieces[3]
335 # Honor the SELinux context if present.
336 context = None
337 for i in mount_flags.split(","):
338 if i.startswith("context="):
339 context = i
Doug Zongker086cbb02011-02-17 15:54:20 -0800340
Tao Baod1de6f32017-03-01 16:38:48 -0800341 mount_point = pieces[1]
342 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2],
343 device=pieces[0], length=length, context=context)
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800344
Daniel Rosenberge6853b02015-06-05 17:59:27 -0700345 # / is used for the system mount point when the root directory is included in
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700346 # system. Other areas assume system is always at "/system" so point /system
347 # at /.
Daniel Rosenberge6853b02015-06-05 17:59:27 -0700348 if system_root_image:
349 assert not d.has_key("/system") and d.has_key("/")
350 d["/system"] = d["/"]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700351 return d
352
353
Doug Zongker37974732010-09-16 17:44:38 -0700354def DumpInfoDict(d):
355 for k, v in sorted(d.items()):
Tao Bao89fbb0f2017-01-10 10:47:58 -0800356 print("%-25s = (%s) %s" % (k, type(v).__name__, v))
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700357
Dan Albert8b72aef2015-03-23 19:13:21 -0700358
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800359def AppendAVBSigningArgs(cmd, partition):
360 """Append signing arguments for avbtool."""
361 # e.g., "--key path/to/signing_key --algorithm SHA256_RSA4096"
362 key_path = OPTIONS.info_dict.get("avb_" + partition + "_key_path")
363 algorithm = OPTIONS.info_dict.get("avb_" + partition + "_algorithm")
364 if key_path and algorithm:
365 cmd.extend(["--key", key_path, "--algorithm", algorithm])
Tao Bao2b6dfd62017-09-27 17:17:43 -0700366 avb_salt = OPTIONS.info_dict.get("avb_salt")
367 # make_vbmeta_image doesn't like "--salt" (and it's not needed).
368 if avb_salt and partition != "vbmeta":
369 cmd.extend(["--salt", avb_salt])
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800370
371
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700372def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
Tao Baod42e97e2016-11-30 12:11:57 -0800373 has_ramdisk=False, two_step_image=False):
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700374 """Build a bootable image from the specified sourcedir.
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700375
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700376 Take a kernel, cmdline, and optionally a ramdisk directory from the input (in
Tao Baod42e97e2016-11-30 12:11:57 -0800377 'sourcedir'), and turn them into a boot image. 'two_step_image' indicates if
378 we are building a two-step special image (i.e. building a recovery image to
379 be loaded into /boot in two-step OTAs).
380
381 Return the image data, or None if sourcedir does not appear to contains files
382 for building the requested image.
383 """
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700384
385 def make_ramdisk():
386 ramdisk_img = tempfile.NamedTemporaryFile()
387
388 if os.access(fs_config_file, os.F_OK):
389 cmd = ["mkbootfs", "-f", fs_config_file,
390 os.path.join(sourcedir, "RAMDISK")]
391 else:
392 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")]
393 p1 = Run(cmd, stdout=subprocess.PIPE)
394 p2 = Run(["minigzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
395
396 p2.wait()
397 p1.wait()
398 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,)
399 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,)
400
401 return ramdisk_img
402
403 if not os.access(os.path.join(sourcedir, "kernel"), os.F_OK):
404 return None
405
406 if has_ramdisk and not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK):
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700407 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700408
Doug Zongkerd5131602012-08-02 14:46:42 -0700409 if info_dict is None:
410 info_dict = OPTIONS.info_dict
411
Doug Zongkereef39442009-04-02 12:14:19 -0700412 img = tempfile.NamedTemporaryFile()
413
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700414 if has_ramdisk:
415 ramdisk_img = make_ramdisk()
Doug Zongkereef39442009-04-02 12:14:19 -0700416
Bjorn Andersson612e2cd2012-11-25 16:53:44 -0800417 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set
418 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg"
419
420 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")]
Doug Zongker38a649f2009-06-17 09:07:09 -0700421
Benoit Fradina45a8682014-07-14 21:00:43 +0200422 fn = os.path.join(sourcedir, "second")
423 if os.access(fn, os.F_OK):
424 cmd.append("--second")
425 cmd.append(fn)
426
Doug Zongker171f1cd2009-06-15 22:36:37 -0700427 fn = os.path.join(sourcedir, "cmdline")
428 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700429 cmd.append("--cmdline")
430 cmd.append(open(fn).read().rstrip("\n"))
431
432 fn = os.path.join(sourcedir, "base")
433 if os.access(fn, os.F_OK):
434 cmd.append("--base")
435 cmd.append(open(fn).read().rstrip("\n"))
436
Ying Wang4de6b5b2010-08-25 14:29:34 -0700437 fn = os.path.join(sourcedir, "pagesize")
438 if os.access(fn, os.F_OK):
439 cmd.append("--pagesize")
440 cmd.append(open(fn).read().rstrip("\n"))
441
David Ng5b00f562012-07-27 18:39:48 -0700442 fn = os.path.join(sourcedir, "tagsaddr")
443 if os.access(fn, os.F_OK):
444 cmd.append("--tags-addr")
445 cmd.append(open(fn).read().rstrip("\n"))
446
447 fn = os.path.join(sourcedir, "ramdisk_offset")
448 if os.access(fn, os.F_OK):
449 cmd.append("--ramdisk_offset")
450 cmd.append(open(fn).read().rstrip("\n"))
451
452 fn = os.path.join(sourcedir, "dt")
453 if os.access(fn, os.F_OK):
454 cmd.append("--dt")
455 cmd.append(fn)
456
Doug Zongkerd5131602012-08-02 14:46:42 -0700457 args = info_dict.get("mkbootimg_args", None)
458 if args and args.strip():
Jianxun Zhang09849492013-04-17 15:19:19 -0700459 cmd.extend(shlex.split(args))
Doug Zongkerd5131602012-08-02 14:46:42 -0700460
Sami Tolvanen3303d902016-03-15 16:49:30 +0000461 args = info_dict.get("mkbootimg_version_args", None)
462 if args and args.strip():
463 cmd.extend(shlex.split(args))
464
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700465 if has_ramdisk:
466 cmd.extend(["--ramdisk", ramdisk_img.name])
467
Tao Baod95e9fd2015-03-29 23:07:41 -0700468 img_unsigned = None
469 if info_dict.get("vboot", None):
470 img_unsigned = tempfile.NamedTemporaryFile()
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700471 cmd.extend(["--output", img_unsigned.name])
Tao Baod95e9fd2015-03-29 23:07:41 -0700472 else:
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700473 cmd.extend(["--output", img.name])
Doug Zongker38a649f2009-06-17 09:07:09 -0700474
Tao Baobf70c3182017-07-11 17:27:55 -0700475 # "boot" or "recovery", without extension.
476 partition_name = os.path.basename(sourcedir).lower()
477
Hridya Valsarajud67d8602018-03-21 12:15:11 -0700478 if (partition_name == "recovery" and
479 info_dict.get("include_recovery_dtbo") == "true"):
480 fn = os.path.join(sourcedir, "recovery_dtbo")
481 cmd.extend(["--recovery_dtbo", fn])
482
Doug Zongker38a649f2009-06-17 09:07:09 -0700483 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700484 p.communicate()
Tao Baobf70c3182017-07-11 17:27:55 -0700485 assert p.returncode == 0, "mkbootimg of %s image failed" % (partition_name,)
Doug Zongkereef39442009-04-02 12:14:19 -0700486
Sami Tolvanen8b3f08b2015-04-07 15:08:59 +0100487 if (info_dict.get("boot_signer", None) == "true" and
488 info_dict.get("verity_key", None)):
Tao Baod42e97e2016-11-30 12:11:57 -0800489 # Hard-code the path as "/boot" for two-step special recovery image (which
490 # will be loaded into /boot during the two-step OTA).
491 if two_step_image:
492 path = "/boot"
493 else:
Tao Baobf70c3182017-07-11 17:27:55 -0700494 path = "/" + partition_name
Baligh Uddin601ddea2015-06-09 15:48:14 -0700495 cmd = [OPTIONS.boot_signer_path]
496 cmd.extend(OPTIONS.boot_signer_args)
497 cmd.extend([path, img.name,
498 info_dict["verity_key"] + ".pk8",
499 info_dict["verity_key"] + ".x509.pem", img.name])
Geremy Condra95ebe7a2014-08-19 17:27:56 -0700500 p = Run(cmd, stdout=subprocess.PIPE)
501 p.communicate()
502 assert p.returncode == 0, "boot_signer of %s image failed" % path
503
Tao Baod95e9fd2015-03-29 23:07:41 -0700504 # Sign the image if vboot is non-empty.
505 elif info_dict.get("vboot", None):
Tao Baobf70c3182017-07-11 17:27:55 -0700506 path = "/" + partition_name
Tao Baod95e9fd2015-03-29 23:07:41 -0700507 img_keyblock = tempfile.NamedTemporaryFile()
Tao Bao4f104d12017-02-17 23:21:31 -0800508 # We have switched from the prebuilt futility binary to using the tool
509 # (futility-host) built from the source. Override the setting in the old
510 # TF.zip.
511 futility = info_dict["futility"]
512 if futility.startswith("prebuilts/"):
513 futility = "futility-host"
514 cmd = [info_dict["vboot_signer_cmd"], futility,
Tao Baod95e9fd2015-03-29 23:07:41 -0700515 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk",
Furquan Shaikh852b8de2015-08-10 11:43:45 -0700516 info_dict["vboot_key"] + ".vbprivk",
517 info_dict["vboot_subkey"] + ".vbprivk",
518 img_keyblock.name,
Tao Baod95e9fd2015-03-29 23:07:41 -0700519 img.name]
520 p = Run(cmd, stdout=subprocess.PIPE)
521 p.communicate()
522 assert p.returncode == 0, "vboot_signer of %s image failed" % path
523
Tao Baof3282b42015-04-01 11:21:55 -0700524 # Clean up the temp files.
525 img_unsigned.close()
526 img_keyblock.close()
527
David Zeuthen8fecb282017-12-01 16:24:01 -0500528 # AVB: if enabled, calculate and add hash to boot.img or recovery.img.
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800529 if info_dict.get("avb_enable") == "true":
Tao Bao3ebfdde2017-05-23 23:06:55 -0700530 avbtool = os.getenv('AVBTOOL') or info_dict["avb_avbtool"]
David Zeuthen8fecb282017-12-01 16:24:01 -0500531 part_size = info_dict[partition_name + "_size"]
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400532 cmd = [avbtool, "add_hash_footer", "--image", img.name,
Tao Baobf70c3182017-07-11 17:27:55 -0700533 "--partition_size", str(part_size), "--partition_name",
534 partition_name]
535 AppendAVBSigningArgs(cmd, partition_name)
David Zeuthen8fecb282017-12-01 16:24:01 -0500536 args = info_dict.get("avb_" + partition_name + "_add_hash_footer_args")
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400537 if args and args.strip():
538 cmd.extend(shlex.split(args))
539 p = Run(cmd, stdout=subprocess.PIPE)
540 p.communicate()
541 assert p.returncode == 0, "avbtool add_hash_footer of %s failed" % (
Tao Baobf70c3182017-07-11 17:27:55 -0700542 partition_name,)
David Zeuthend995f4b2016-01-29 16:59:17 -0500543
544 img.seek(os.SEEK_SET, 0)
545 data = img.read()
546
547 if has_ramdisk:
548 ramdisk_img.close()
549 img.close()
550
551 return data
552
553
Doug Zongkerd5131602012-08-02 14:46:42 -0700554def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,
Tao Baod42e97e2016-11-30 12:11:57 -0800555 info_dict=None, two_step_image=False):
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700556 """Return a File object with the desired bootable image.
557
558 Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 'prebuilt_name',
559 otherwise look for it under 'unpack_dir'/IMAGES, otherwise construct it from
560 the source files in 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700561
Doug Zongker55d93282011-01-25 17:03:34 -0800562 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
563 if os.path.exists(prebuilt_path):
Tao Bao89fbb0f2017-01-10 10:47:58 -0800564 print("using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,))
Doug Zongker55d93282011-01-25 17:03:34 -0800565 return File.FromLocalFile(name, prebuilt_path)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700566
567 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name)
568 if os.path.exists(prebuilt_path):
Tao Bao89fbb0f2017-01-10 10:47:58 -0800569 print("using prebuilt %s from IMAGES..." % (prebuilt_name,))
Doug Zongker6f1d0312014-08-22 08:07:12 -0700570 return File.FromLocalFile(name, prebuilt_path)
571
Tao Bao89fbb0f2017-01-10 10:47:58 -0800572 print("building image from target_files %s..." % (tree_subdir,))
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700573
574 if info_dict is None:
575 info_dict = OPTIONS.info_dict
576
577 # With system_root_image == "true", we don't pack ramdisk into the boot image.
Daniel Rosenberg40ef35b2015-11-10 19:21:34 -0800578 # Unless "recovery_as_boot" is specified, in which case we carry the ramdisk
579 # for recovery.
580 has_ramdisk = (info_dict.get("system_root_image") != "true" or
581 prebuilt_name != "boot.img" or
582 info_dict.get("recovery_as_boot") == "true")
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700583
Doug Zongker6f1d0312014-08-22 08:07:12 -0700584 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt"
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400585 data = _BuildBootableImage(os.path.join(unpack_dir, tree_subdir),
586 os.path.join(unpack_dir, fs_config),
Tao Baod42e97e2016-11-30 12:11:57 -0800587 info_dict, has_ramdisk, two_step_image)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700588 if data:
589 return File(name, data)
590 return None
Doug Zongker55d93282011-01-25 17:03:34 -0800591
Doug Zongkereef39442009-04-02 12:14:19 -0700592
Narayan Kamatha07bf042017-08-14 14:49:21 +0100593def Gunzip(in_filename, out_filename):
594 """Gunzip the given gzip compressed file to a given output file.
595 """
596 with gzip.open(in_filename, "rb") as in_file, open(out_filename, "wb") as out_file:
597 shutil.copyfileobj(in_file, out_file)
598
599
Doug Zongker75f17362009-12-08 13:46:44 -0800600def UnzipTemp(filename, pattern=None):
Tao Bao1c830bf2017-12-25 10:43:47 -0800601 """Unzips the given archive into a temporary directory and returns the name.
Doug Zongker55d93282011-01-25 17:03:34 -0800602
Tao Bao1c830bf2017-12-25 10:43:47 -0800603 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a temp dir,
604 then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
Doug Zongker55d93282011-01-25 17:03:34 -0800605
Tao Bao1c830bf2017-12-25 10:43:47 -0800606 Returns:
Tao Baodba59ee2018-01-09 13:21:02 -0800607 The name of the temporary directory.
Doug Zongker55d93282011-01-25 17:03:34 -0800608 """
Doug Zongkereef39442009-04-02 12:14:19 -0700609
Doug Zongker55d93282011-01-25 17:03:34 -0800610 def unzip_to_dir(filename, dirname):
611 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
612 if pattern is not None:
Tao Bao6b0b2f92017-03-05 11:38:11 -0800613 cmd.extend(pattern)
Doug Zongker55d93282011-01-25 17:03:34 -0800614 p = Run(cmd, stdout=subprocess.PIPE)
615 p.communicate()
616 if p.returncode != 0:
617 raise ExternalError("failed to unzip input target-files \"%s\"" %
618 (filename,))
619
Tao Bao1c830bf2017-12-25 10:43:47 -0800620 tmp = MakeTempDir(prefix="targetfiles-")
Doug Zongker55d93282011-01-25 17:03:34 -0800621 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
622 if m:
623 unzip_to_dir(m.group(1), tmp)
624 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
625 filename = m.group(1)
626 else:
627 unzip_to_dir(filename, tmp)
628
Tao Baodba59ee2018-01-09 13:21:02 -0800629 return tmp
Doug Zongkereef39442009-04-02 12:14:19 -0700630
631
Tao Baoe709b092018-02-07 12:40:00 -0800632def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks):
Tao Baoc765cca2018-01-31 17:32:40 -0800633 """Returns a SparseImage object suitable for passing to BlockImageDiff.
634
635 This function loads the specified sparse image from the given path, and
636 performs additional processing for OTA purpose. For example, it always adds
637 block 0 to clobbered blocks list. It also detects files that cannot be
638 reconstructed from the block list, for whom we should avoid applying imgdiff.
639
640 Args:
641 which: The partition name, which must be "system" or "vendor".
642 tmpdir: The directory that contains the prebuilt image and block map file.
643 input_zip: The target-files ZIP archive.
Tao Baoe709b092018-02-07 12:40:00 -0800644 allow_shared_blocks: Whether having shared blocks is allowed.
Tao Baoc765cca2018-01-31 17:32:40 -0800645
646 Returns:
647 A SparseImage object, with file_map info loaded.
648 """
649 assert which in ("system", "vendor")
650
651 path = os.path.join(tmpdir, "IMAGES", which + ".img")
652 mappath = os.path.join(tmpdir, "IMAGES", which + ".map")
653
654 # The image and map files must have been created prior to calling
655 # ota_from_target_files.py (since LMP).
656 assert os.path.exists(path) and os.path.exists(mappath)
657
658 # In ext4 filesystems, block 0 might be changed even being mounted R/O. We add
659 # it to clobbered_blocks so that it will be written to the target
660 # unconditionally. Note that they are still part of care_map. (Bug: 20939131)
661 clobbered_blocks = "0"
662
Tao Baoe709b092018-02-07 12:40:00 -0800663 image = sparse_img.SparseImage(path, mappath, clobbered_blocks,
664 allow_shared_blocks=allow_shared_blocks)
Tao Baoc765cca2018-01-31 17:32:40 -0800665
666 # block.map may contain less blocks, because mke2fs may skip allocating blocks
667 # if they contain all zeros. We can't reconstruct such a file from its block
668 # list. Tag such entries accordingly. (Bug: 65213616)
669 for entry in image.file_map:
670 # "/system/framework/am.jar" => "SYSTEM/framework/am.jar".
671 arcname = string.replace(entry, which, which.upper(), 1)[1:]
672 # Skip artificial names, such as "__ZERO", "__NONZERO-1".
673 if arcname not in input_zip.namelist():
674 continue
675
676 info = input_zip.getinfo(arcname)
677 ranges = image.file_map[entry]
Tao Baoe709b092018-02-07 12:40:00 -0800678
679 # If a RangeSet has been tagged as using shared blocks while loading the
680 # image, its block list must be already incomplete due to that reason. Don't
681 # give it 'incomplete' tag to avoid messing up the imgdiff stats.
682 if ranges.extra.get('uses_shared_blocks'):
683 continue
684
Tao Baoc765cca2018-01-31 17:32:40 -0800685 if RoundUpTo4K(info.file_size) > ranges.size() * 4096:
686 ranges.extra['incomplete'] = True
687
688 return image
689
690
Doug Zongkereef39442009-04-02 12:14:19 -0700691def GetKeyPasswords(keylist):
692 """Given a list of keys, prompt the user to enter passwords for
693 those which require them. Return a {key: password} dict. password
694 will be None if the key has no password."""
695
Doug Zongker8ce7c252009-05-22 13:34:54 -0700696 no_passwords = []
697 need_passwords = []
T.R. Fullhart37e10522013-03-18 10:31:26 -0700698 key_passwords = {}
Doug Zongkereef39442009-04-02 12:14:19 -0700699 devnull = open("/dev/null", "w+b")
700 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800701 # We don't need a password for things that aren't really keys.
702 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700703 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700704 continue
705
T.R. Fullhart37e10522013-03-18 10:31:26 -0700706 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
Doug Zongker602a84e2009-06-18 08:35:12 -0700707 "-inform", "DER", "-nocrypt"],
708 stdin=devnull.fileno(),
709 stdout=devnull.fileno(),
710 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700711 p.communicate()
712 if p.returncode == 0:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700713 # Definitely an unencrypted key.
Doug Zongker8ce7c252009-05-22 13:34:54 -0700714 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700715 else:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700716 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
717 "-inform", "DER", "-passin", "pass:"],
718 stdin=devnull.fileno(),
719 stdout=devnull.fileno(),
720 stderr=subprocess.PIPE)
Dan Albert8b72aef2015-03-23 19:13:21 -0700721 _, stderr = p.communicate()
T.R. Fullhart37e10522013-03-18 10:31:26 -0700722 if p.returncode == 0:
723 # Encrypted key with empty string as password.
724 key_passwords[k] = ''
725 elif stderr.startswith('Error decrypting key'):
726 # Definitely encrypted key.
727 # It would have said "Error reading key" if it didn't parse correctly.
728 need_passwords.append(k)
729 else:
730 # Potentially, a type of key that openssl doesn't understand.
731 # We'll let the routines in signapk.jar handle it.
732 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700733 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700734
T.R. Fullhart37e10522013-03-18 10:31:26 -0700735 key_passwords.update(PasswordManager().GetPasswords(need_passwords))
Doug Zongker8ce7c252009-05-22 13:34:54 -0700736 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700737 return key_passwords
738
739
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800740def GetMinSdkVersion(apk_name):
741 """Get the minSdkVersion delared in the APK. This can be both a decimal number
742 (API Level) or a codename.
743 """
744
745 p = Run(["aapt", "dump", "badging", apk_name], stdout=subprocess.PIPE)
746 output, err = p.communicate()
747 if err:
748 raise ExternalError("Failed to obtain minSdkVersion: aapt return code %s"
749 % (p.returncode,))
750
751 for line in output.split("\n"):
752 # Looking for lines such as sdkVersion:'23' or sdkVersion:'M'
753 m = re.match(r'sdkVersion:\'([^\']*)\'', line)
754 if m:
755 return m.group(1)
756 raise ExternalError("No minSdkVersion returned by aapt")
757
758
759def GetMinSdkVersionInt(apk_name, codename_to_api_level_map):
760 """Get the minSdkVersion declared in the APK as a number (API Level). If
761 minSdkVersion is set to a codename, it is translated to a number using the
762 provided map.
763 """
764
765 version = GetMinSdkVersion(apk_name)
766 try:
767 return int(version)
768 except ValueError:
769 # Not a decimal number. Codename?
770 if version in codename_to_api_level_map:
771 return codename_to_api_level_map[version]
772 else:
773 raise ExternalError("Unknown minSdkVersion: '%s'. Known codenames: %s"
774 % (version, codename_to_api_level_map))
775
776
777def SignFile(input_name, output_name, key, password, min_api_level=None,
778 codename_to_api_level_map=dict(),
779 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700780 """Sign the input_name zip/jar/apk, producing output_name. Use the
781 given key and password (the latter may be None if the key does not
782 have a password.
783
Doug Zongker951495f2009-08-14 12:44:19 -0700784 If whole_file is true, use the "-w" option to SignApk to embed a
785 signature that covers the whole file in the archive comment of the
786 zip file.
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800787
788 min_api_level is the API Level (int) of the oldest platform this file may end
789 up on. If not specified for an APK, the API Level is obtained by interpreting
790 the minSdkVersion attribute of the APK's AndroidManifest.xml.
791
792 codename_to_api_level_map is needed to translate the codename which may be
793 encountered as the APK's minSdkVersion.
Doug Zongkereef39442009-04-02 12:14:19 -0700794 """
Doug Zongker951495f2009-08-14 12:44:19 -0700795
Alex Klyubin9667b182015-12-10 13:38:50 -0800796 java_library_path = os.path.join(
797 OPTIONS.search_path, OPTIONS.signapk_shared_library_path)
798
Tao Baoe95540e2016-11-08 12:08:53 -0800799 cmd = ([OPTIONS.java_path] + OPTIONS.java_args +
800 ["-Djava.library.path=" + java_library_path,
801 "-jar", os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] +
802 OPTIONS.extra_signapk_args)
Doug Zongker951495f2009-08-14 12:44:19 -0700803 if whole_file:
804 cmd.append("-w")
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800805
806 min_sdk_version = min_api_level
807 if min_sdk_version is None:
808 if not whole_file:
809 min_sdk_version = GetMinSdkVersionInt(
810 input_name, codename_to_api_level_map)
811 if min_sdk_version is not None:
812 cmd.extend(["--min-sdk-version", str(min_sdk_version)])
813
T.R. Fullhart37e10522013-03-18 10:31:26 -0700814 cmd.extend([key + OPTIONS.public_key_suffix,
815 key + OPTIONS.private_key_suffix,
Alex Klyubineb756d72015-12-04 09:21:08 -0800816 input_name, output_name])
Doug Zongker951495f2009-08-14 12:44:19 -0700817
818 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700819 if password is not None:
820 password += "\n"
821 p.communicate(password)
822 if p.returncode != 0:
823 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
824
Doug Zongkereef39442009-04-02 12:14:19 -0700825
Doug Zongker37974732010-09-16 17:44:38 -0700826def CheckSize(data, target, info_dict):
Tao Bao9dd909e2017-11-14 11:27:32 -0800827 """Checks the data string passed against the max size limit.
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700828
Tao Bao9dd909e2017-11-14 11:27:32 -0800829 For non-AVB images, raise exception if the data is too big. Print a warning
830 if the data is nearing the maximum size.
831
832 For AVB images, the actual image size should be identical to the limit.
833
834 Args:
835 data: A string that contains all the data for the partition.
836 target: The partition name. The ".img" suffix is optional.
837 info_dict: The dict to be looked up for relevant info.
838 """
Dan Albert8b72aef2015-03-23 19:13:21 -0700839 if target.endswith(".img"):
840 target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700841 mount_point = "/" + target
842
Ying Wangf8824af2014-06-03 14:07:27 -0700843 fs_type = None
844 limit = None
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700845 if info_dict["fstab"]:
Dan Albert8b72aef2015-03-23 19:13:21 -0700846 if mount_point == "/userdata":
847 mount_point = "/data"
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700848 p = info_dict["fstab"][mount_point]
849 fs_type = p.fs_type
Andrew Boie0f9aec82012-02-14 09:32:52 -0800850 device = p.device
851 if "/" in device:
852 device = device[device.rfind("/")+1:]
853 limit = info_dict.get(device + "_size", None)
Dan Albert8b72aef2015-03-23 19:13:21 -0700854 if not fs_type or not limit:
855 return
Doug Zongkereef39442009-04-02 12:14:19 -0700856
Andrew Boie0f9aec82012-02-14 09:32:52 -0800857 size = len(data)
Tao Bao9dd909e2017-11-14 11:27:32 -0800858 # target could be 'userdata' or 'cache'. They should follow the non-AVB image
859 # path.
860 if info_dict.get("avb_enable") == "true" and target in AVB_PARTITIONS:
861 if size != limit:
862 raise ExternalError(
863 "Mismatching image size for %s: expected %d actual %d" % (
864 target, limit, size))
865 else:
866 pct = float(size) * 100.0 / limit
867 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
868 if pct >= 99.0:
869 raise ExternalError(msg)
870 elif pct >= 95.0:
871 print("\n WARNING: %s\n" % (msg,))
872 elif OPTIONS.verbose:
873 print(" ", msg)
Doug Zongkereef39442009-04-02 12:14:19 -0700874
875
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800876def ReadApkCerts(tf_zip):
Tao Bao818ddf52018-01-05 11:17:34 -0800877 """Parses the APK certs info from a given target-files zip.
878
879 Given a target-files ZipFile, parses the META/apkcerts.txt entry and returns a
880 tuple with the following elements: (1) a dictionary that maps packages to
881 certs (based on the "certificate" and "private_key" attributes in the file;
882 (2) a string representing the extension of compressed APKs in the target files
883 (e.g ".gz", ".bro").
884
885 Args:
886 tf_zip: The input target_files ZipFile (already open).
887
888 Returns:
889 (certmap, ext): certmap is a dictionary that maps packages to certs; ext is
890 the extension string of compressed APKs (e.g. ".gz"), or None if there's
891 no compressed APKs.
892 """
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800893 certmap = {}
Narayan Kamatha07bf042017-08-14 14:49:21 +0100894 compressed_extension = None
895
Tao Bao0f990332017-09-08 19:02:54 -0700896 # META/apkcerts.txt contains the info for _all_ the packages known at build
897 # time. Filter out the ones that are not installed.
898 installed_files = set()
899 for name in tf_zip.namelist():
900 basename = os.path.basename(name)
901 if basename:
902 installed_files.add(basename)
903
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800904 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
905 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700906 if not line:
907 continue
Tao Bao818ddf52018-01-05 11:17:34 -0800908 m = re.match(
909 r'^name="(?P<NAME>.*)"\s+certificate="(?P<CERT>.*)"\s+'
910 r'private_key="(?P<PRIVKEY>.*?)"(\s+compressed="(?P<COMPRESSED>.*)")?$',
911 line)
912 if not m:
913 continue
Narayan Kamatha07bf042017-08-14 14:49:21 +0100914
Tao Bao818ddf52018-01-05 11:17:34 -0800915 matches = m.groupdict()
916 cert = matches["CERT"]
917 privkey = matches["PRIVKEY"]
918 name = matches["NAME"]
919 this_compressed_extension = matches["COMPRESSED"]
920
921 public_key_suffix_len = len(OPTIONS.public_key_suffix)
922 private_key_suffix_len = len(OPTIONS.private_key_suffix)
923 if cert in SPECIAL_CERT_STRINGS and not privkey:
924 certmap[name] = cert
925 elif (cert.endswith(OPTIONS.public_key_suffix) and
926 privkey.endswith(OPTIONS.private_key_suffix) and
927 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]):
928 certmap[name] = cert[:-public_key_suffix_len]
929 else:
930 raise ValueError("Failed to parse line from apkcerts.txt:\n" + line)
931
932 if not this_compressed_extension:
933 continue
934
935 # Only count the installed files.
936 filename = name + '.' + this_compressed_extension
937 if filename not in installed_files:
938 continue
939
940 # Make sure that all the values in the compression map have the same
941 # extension. We don't support multiple compression methods in the same
942 # system image.
943 if compressed_extension:
944 if this_compressed_extension != compressed_extension:
945 raise ValueError(
946 "Multiple compressed extensions: {} vs {}".format(
947 compressed_extension, this_compressed_extension))
948 else:
949 compressed_extension = this_compressed_extension
950
951 return (certmap,
952 ("." + compressed_extension) if compressed_extension else None)
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800953
954
Doug Zongkereef39442009-04-02 12:14:19 -0700955COMMON_DOCSTRING = """
956 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700957 Prepend <dir>/bin to the list of places to search for binaries
958 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700959
Doug Zongker05d3dea2009-06-22 11:32:31 -0700960 -s (--device_specific) <file>
961 Path to the python module containing device-specific
962 releasetools code.
963
Doug Zongker8bec09e2009-11-30 15:37:14 -0800964 -x (--extra) <key=value>
965 Add a key/value pair to the 'extras' dict, which device-specific
966 extension code may look at.
967
Doug Zongkereef39442009-04-02 12:14:19 -0700968 -v (--verbose)
969 Show command lines being executed.
970
971 -h (--help)
972 Display this usage message and exit.
973"""
974
975def Usage(docstring):
Tao Bao89fbb0f2017-01-10 10:47:58 -0800976 print(docstring.rstrip("\n"))
977 print(COMMON_DOCSTRING)
Doug Zongkereef39442009-04-02 12:14:19 -0700978
979
980def ParseOptions(argv,
981 docstring,
982 extra_opts="", extra_long_opts=(),
983 extra_option_handler=None):
984 """Parse the options in argv and return any arguments that aren't
985 flags. docstring is the calling module's docstring, to be displayed
986 for errors and -h. extra_opts and extra_long_opts are for flags
987 defined by the caller, which are processed by passing them to
988 extra_option_handler."""
989
990 try:
991 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800992 argv, "hvp:s:x:" + extra_opts,
Alex Klyubin9667b182015-12-10 13:38:50 -0800993 ["help", "verbose", "path=", "signapk_path=",
994 "signapk_shared_library_path=", "extra_signapk_args=",
Baligh Uddinbdc2e312014-09-05 17:36:20 -0700995 "java_path=", "java_args=", "public_key_suffix=",
Baligh Uddin601ddea2015-06-09 15:48:14 -0700996 "private_key_suffix=", "boot_signer_path=", "boot_signer_args=",
997 "verity_signer_path=", "verity_signer_args=", "device_specific=",
Baligh Uddine2048682014-11-20 09:52:05 -0800998 "extra="] +
T.R. Fullhart37e10522013-03-18 10:31:26 -0700999 list(extra_long_opts))
Dan Albert8b72aef2015-03-23 19:13:21 -07001000 except getopt.GetoptError as err:
Doug Zongkereef39442009-04-02 12:14:19 -07001001 Usage(docstring)
Tao Bao89fbb0f2017-01-10 10:47:58 -08001002 print("**", str(err), "**")
Doug Zongkereef39442009-04-02 12:14:19 -07001003 sys.exit(2)
1004
Doug Zongkereef39442009-04-02 12:14:19 -07001005 for o, a in opts:
1006 if o in ("-h", "--help"):
1007 Usage(docstring)
1008 sys.exit()
1009 elif o in ("-v", "--verbose"):
1010 OPTIONS.verbose = True
1011 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -07001012 OPTIONS.search_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -07001013 elif o in ("--signapk_path",):
1014 OPTIONS.signapk_path = a
Alex Klyubin9667b182015-12-10 13:38:50 -08001015 elif o in ("--signapk_shared_library_path",):
1016 OPTIONS.signapk_shared_library_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -07001017 elif o in ("--extra_signapk_args",):
1018 OPTIONS.extra_signapk_args = shlex.split(a)
1019 elif o in ("--java_path",):
1020 OPTIONS.java_path = a
Baligh Uddin339ee492014-09-05 11:18:07 -07001021 elif o in ("--java_args",):
Tao Baoe95540e2016-11-08 12:08:53 -08001022 OPTIONS.java_args = shlex.split(a)
T.R. Fullhart37e10522013-03-18 10:31:26 -07001023 elif o in ("--public_key_suffix",):
1024 OPTIONS.public_key_suffix = a
1025 elif o in ("--private_key_suffix",):
1026 OPTIONS.private_key_suffix = a
Baligh Uddine2048682014-11-20 09:52:05 -08001027 elif o in ("--boot_signer_path",):
1028 OPTIONS.boot_signer_path = a
Baligh Uddin601ddea2015-06-09 15:48:14 -07001029 elif o in ("--boot_signer_args",):
1030 OPTIONS.boot_signer_args = shlex.split(a)
1031 elif o in ("--verity_signer_path",):
1032 OPTIONS.verity_signer_path = a
1033 elif o in ("--verity_signer_args",):
1034 OPTIONS.verity_signer_args = shlex.split(a)
Doug Zongker05d3dea2009-06-22 11:32:31 -07001035 elif o in ("-s", "--device_specific"):
1036 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -08001037 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -08001038 key, value = a.split("=", 1)
1039 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -07001040 else:
1041 if extra_option_handler is None or not extra_option_handler(o, a):
1042 assert False, "unknown option \"%s\"" % (o,)
1043
Doug Zongker85448772014-09-09 14:59:20 -07001044 if OPTIONS.search_path:
1045 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
1046 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -07001047
1048 return args
1049
1050
Tao Bao4c851b12016-09-19 13:54:38 -07001051def MakeTempFile(prefix='tmp', suffix=''):
Doug Zongkerfc44a512014-08-26 13:10:25 -07001052 """Make a temp file and add it to the list of things to be deleted
1053 when Cleanup() is called. Return the filename."""
1054 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix)
1055 os.close(fd)
1056 OPTIONS.tempfiles.append(fn)
1057 return fn
1058
1059
Tao Bao1c830bf2017-12-25 10:43:47 -08001060def MakeTempDir(prefix='tmp', suffix=''):
1061 """Makes a temporary dir that will be cleaned up with a call to Cleanup().
1062
1063 Returns:
1064 The absolute pathname of the new directory.
1065 """
1066 dir_name = tempfile.mkdtemp(suffix=suffix, prefix=prefix)
1067 OPTIONS.tempfiles.append(dir_name)
1068 return dir_name
1069
1070
Doug Zongkereef39442009-04-02 12:14:19 -07001071def Cleanup():
1072 for i in OPTIONS.tempfiles:
1073 if os.path.isdir(i):
Tao Bao1c830bf2017-12-25 10:43:47 -08001074 shutil.rmtree(i, ignore_errors=True)
Doug Zongkereef39442009-04-02 12:14:19 -07001075 else:
1076 os.remove(i)
Tao Bao1c830bf2017-12-25 10:43:47 -08001077 del OPTIONS.tempfiles[:]
Doug Zongker8ce7c252009-05-22 13:34:54 -07001078
1079
1080class PasswordManager(object):
1081 def __init__(self):
1082 self.editor = os.getenv("EDITOR", None)
1083 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
1084
1085 def GetPasswords(self, items):
1086 """Get passwords corresponding to each string in 'items',
1087 returning a dict. (The dict may have keys in addition to the
1088 values in 'items'.)
1089
1090 Uses the passwords in $ANDROID_PW_FILE if available, letting the
1091 user edit that file to add more needed passwords. If no editor is
1092 available, or $ANDROID_PW_FILE isn't define, prompts the user
1093 interactively in the ordinary way.
1094 """
1095
1096 current = self.ReadFile()
1097
1098 first = True
1099 while True:
1100 missing = []
1101 for i in items:
1102 if i not in current or not current[i]:
1103 missing.append(i)
1104 # Are all the passwords already in the file?
Dan Albert8b72aef2015-03-23 19:13:21 -07001105 if not missing:
1106 return current
Doug Zongker8ce7c252009-05-22 13:34:54 -07001107
1108 for i in missing:
1109 current[i] = ""
1110
1111 if not first:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001112 print("key file %s still missing some passwords." % (self.pwfile,))
Doug Zongker8ce7c252009-05-22 13:34:54 -07001113 answer = raw_input("try to edit again? [y]> ").strip()
1114 if answer and answer[0] not in 'yY':
1115 raise RuntimeError("key passwords unavailable")
1116 first = False
1117
1118 current = self.UpdateAndReadFile(current)
1119
Dan Albert8b72aef2015-03-23 19:13:21 -07001120 def PromptResult(self, current): # pylint: disable=no-self-use
Doug Zongker8ce7c252009-05-22 13:34:54 -07001121 """Prompt the user to enter a value (password) for each key in
1122 'current' whose value is fales. Returns a new dict with all the
1123 values.
1124 """
1125 result = {}
1126 for k, v in sorted(current.iteritems()):
1127 if v:
1128 result[k] = v
1129 else:
1130 while True:
Dan Albert8b72aef2015-03-23 19:13:21 -07001131 result[k] = getpass.getpass(
1132 "Enter password for %s key> " % k).strip()
1133 if result[k]:
1134 break
Doug Zongker8ce7c252009-05-22 13:34:54 -07001135 return result
1136
1137 def UpdateAndReadFile(self, current):
1138 if not self.editor or not self.pwfile:
1139 return self.PromptResult(current)
1140
1141 f = open(self.pwfile, "w")
Dan Albert8b72aef2015-03-23 19:13:21 -07001142 os.chmod(self.pwfile, 0o600)
Doug Zongker8ce7c252009-05-22 13:34:54 -07001143 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
1144 f.write("# (Additional spaces are harmless.)\n\n")
1145
1146 first_line = None
Dan Albert8b72aef2015-03-23 19:13:21 -07001147 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()])
1148 for i, (_, k, v) in enumerate(sorted_list):
Doug Zongker8ce7c252009-05-22 13:34:54 -07001149 f.write("[[[ %s ]]] %s\n" % (v, k))
1150 if not v and first_line is None:
1151 # position cursor on first line with no password.
1152 first_line = i + 4
1153 f.close()
1154
1155 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
1156 _, _ = p.communicate()
1157
1158 return self.ReadFile()
1159
1160 def ReadFile(self):
1161 result = {}
Dan Albert8b72aef2015-03-23 19:13:21 -07001162 if self.pwfile is None:
1163 return result
Doug Zongker8ce7c252009-05-22 13:34:54 -07001164 try:
1165 f = open(self.pwfile, "r")
1166 for line in f:
1167 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -07001168 if not line or line[0] == '#':
1169 continue
Doug Zongker8ce7c252009-05-22 13:34:54 -07001170 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
1171 if not m:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001172 print("failed to parse password file: ", line)
Doug Zongker8ce7c252009-05-22 13:34:54 -07001173 else:
1174 result[m.group(2)] = m.group(1)
1175 f.close()
Dan Albert8b72aef2015-03-23 19:13:21 -07001176 except IOError as e:
Doug Zongker8ce7c252009-05-22 13:34:54 -07001177 if e.errno != errno.ENOENT:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001178 print("error reading password file: ", str(e))
Doug Zongker8ce7c252009-05-22 13:34:54 -07001179 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -07001180
1181
Dan Albert8e0178d2015-01-27 15:53:15 -08001182def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
1183 compress_type=None):
1184 import datetime
1185
1186 # http://b/18015246
1187 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required
1188 # for files larger than 2GiB. We can work around this by adjusting their
1189 # limit. Note that `zipfile.writestr()` will not work for strings larger than
1190 # 2GiB. The Python interpreter sometimes rejects strings that large (though
1191 # it isn't clear to me exactly what circumstances cause this).
1192 # `zipfile.write()` must be used directly to work around this.
1193 #
1194 # This mess can be avoided if we port to python3.
1195 saved_zip64_limit = zipfile.ZIP64_LIMIT
1196 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1197
1198 if compress_type is None:
1199 compress_type = zip_file.compression
1200 if arcname is None:
1201 arcname = filename
1202
1203 saved_stat = os.stat(filename)
1204
1205 try:
1206 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the
1207 # file to be zipped and reset it when we're done.
1208 os.chmod(filename, perms)
1209
1210 # Use a fixed timestamp so the output is repeatable.
1211 epoch = datetime.datetime.fromtimestamp(0)
1212 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
1213 os.utime(filename, (timestamp, timestamp))
1214
1215 zip_file.write(filename, arcname=arcname, compress_type=compress_type)
1216 finally:
1217 os.chmod(filename, saved_stat.st_mode)
1218 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime))
1219 zipfile.ZIP64_LIMIT = saved_zip64_limit
1220
1221
Tao Bao58c1b962015-05-20 09:32:18 -07001222def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
Tao Baof3282b42015-04-01 11:21:55 -07001223 compress_type=None):
1224 """Wrap zipfile.writestr() function to work around the zip64 limit.
1225
1226 Even with the ZIP64_LIMIT workaround, it won't allow writing a string
1227 longer than 2GiB. It gives 'OverflowError: size does not fit in an int'
1228 when calling crc32(bytes).
1229
1230 But it still works fine to write a shorter string into a large zip file.
1231 We should use ZipWrite() whenever possible, and only use ZipWriteStr()
1232 when we know the string won't be too long.
1233 """
1234
1235 saved_zip64_limit = zipfile.ZIP64_LIMIT
1236 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1237
1238 if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
1239 zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)
Dan Albert8b72aef2015-03-23 19:13:21 -07001240 zinfo.compress_type = zip_file.compression
Tao Bao58c1b962015-05-20 09:32:18 -07001241 if perms is None:
Tao Bao2a410582015-07-10 17:18:23 -07001242 perms = 0o100644
Geremy Condra36bd3652014-02-06 19:45:10 -08001243 else:
Tao Baof3282b42015-04-01 11:21:55 -07001244 zinfo = zinfo_or_arcname
1245
1246 # If compress_type is given, it overrides the value in zinfo.
1247 if compress_type is not None:
1248 zinfo.compress_type = compress_type
1249
Tao Bao58c1b962015-05-20 09:32:18 -07001250 # If perms is given, it has a priority.
1251 if perms is not None:
Tao Bao2a410582015-07-10 17:18:23 -07001252 # If perms doesn't set the file type, mark it as a regular file.
1253 if perms & 0o770000 == 0:
1254 perms |= 0o100000
Tao Bao58c1b962015-05-20 09:32:18 -07001255 zinfo.external_attr = perms << 16
1256
Tao Baof3282b42015-04-01 11:21:55 -07001257 # Use a fixed timestamp so the output is repeatable.
Tao Baof3282b42015-04-01 11:21:55 -07001258 zinfo.date_time = (2009, 1, 1, 0, 0, 0)
1259
Dan Albert8b72aef2015-03-23 19:13:21 -07001260 zip_file.writestr(zinfo, data)
Tao Baof3282b42015-04-01 11:21:55 -07001261 zipfile.ZIP64_LIMIT = saved_zip64_limit
1262
1263
Tao Bao89d7ab22017-12-14 17:05:33 -08001264def ZipDelete(zip_filename, entries):
1265 """Deletes entries from a ZIP file.
1266
1267 Since deleting entries from a ZIP file is not supported, it shells out to
1268 'zip -d'.
1269
1270 Args:
1271 zip_filename: The name of the ZIP file.
1272 entries: The name of the entry, or the list of names to be deleted.
1273
1274 Raises:
1275 AssertionError: In case of non-zero return from 'zip'.
1276 """
1277 if isinstance(entries, basestring):
1278 entries = [entries]
1279 cmd = ["zip", "-d", zip_filename] + entries
1280 proc = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1281 stdoutdata, _ = proc.communicate()
1282 assert proc.returncode == 0, "Failed to delete %s:\n%s" % (entries,
1283 stdoutdata)
1284
1285
Tao Baof3282b42015-04-01 11:21:55 -07001286def ZipClose(zip_file):
1287 # http://b/18015246
1288 # zipfile also refers to ZIP64_LIMIT during close() when it writes out the
1289 # central directory.
1290 saved_zip64_limit = zipfile.ZIP64_LIMIT
1291 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1292
1293 zip_file.close()
1294
1295 zipfile.ZIP64_LIMIT = saved_zip64_limit
Doug Zongker05d3dea2009-06-22 11:32:31 -07001296
1297
1298class DeviceSpecificParams(object):
1299 module = None
1300 def __init__(self, **kwargs):
1301 """Keyword arguments to the constructor become attributes of this
1302 object, which is passed to all functions in the device-specific
1303 module."""
1304 for k, v in kwargs.iteritems():
1305 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -08001306 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -07001307
1308 if self.module is None:
1309 path = OPTIONS.device_specific
Dan Albert8b72aef2015-03-23 19:13:21 -07001310 if not path:
1311 return
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001312 try:
1313 if os.path.isdir(path):
1314 info = imp.find_module("releasetools", [path])
1315 else:
1316 d, f = os.path.split(path)
1317 b, x = os.path.splitext(f)
1318 if x == ".py":
1319 f = b
1320 info = imp.find_module(f, [d])
Tao Bao89fbb0f2017-01-10 10:47:58 -08001321 print("loaded device-specific extensions from", path)
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001322 self.module = imp.load_module("device_specific", *info)
1323 except ImportError:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001324 print("unable to load device-specific module; assuming none")
Doug Zongker05d3dea2009-06-22 11:32:31 -07001325
1326 def _DoCall(self, function_name, *args, **kwargs):
1327 """Call the named function in the device-specific module, passing
1328 the given args and kwargs. The first argument to the call will be
1329 the DeviceSpecific object itself. If there is no module, or the
1330 module does not define the function, return the value of the
1331 'default' kwarg (which itself defaults to None)."""
1332 if self.module is None or not hasattr(self.module, function_name):
1333 return kwargs.get("default", None)
1334 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
1335
1336 def FullOTA_Assertions(self):
1337 """Called after emitting the block of assertions at the top of a
1338 full OTA package. Implementations can add whatever additional
1339 assertions they like."""
1340 return self._DoCall("FullOTA_Assertions")
1341
Doug Zongkere5ff5902012-01-17 10:55:37 -08001342 def FullOTA_InstallBegin(self):
1343 """Called at the start of full OTA installation."""
1344 return self._DoCall("FullOTA_InstallBegin")
1345
Doug Zongker05d3dea2009-06-22 11:32:31 -07001346 def FullOTA_InstallEnd(self):
1347 """Called at the end of full OTA installation; typically this is
1348 used to install the image for the device's baseband processor."""
1349 return self._DoCall("FullOTA_InstallEnd")
1350
1351 def IncrementalOTA_Assertions(self):
1352 """Called after emitting the block of assertions at the top of an
1353 incremental OTA package. Implementations can add whatever
1354 additional assertions they like."""
1355 return self._DoCall("IncrementalOTA_Assertions")
1356
Doug Zongkere5ff5902012-01-17 10:55:37 -08001357 def IncrementalOTA_VerifyBegin(self):
1358 """Called at the start of the verification phase of incremental
1359 OTA installation; additional checks can be placed here to abort
1360 the script before any changes are made."""
1361 return self._DoCall("IncrementalOTA_VerifyBegin")
1362
Doug Zongker05d3dea2009-06-22 11:32:31 -07001363 def IncrementalOTA_VerifyEnd(self):
1364 """Called at the end of the verification phase of incremental OTA
1365 installation; additional checks can be placed here to abort the
1366 script before any changes are made."""
1367 return self._DoCall("IncrementalOTA_VerifyEnd")
1368
Doug Zongkere5ff5902012-01-17 10:55:37 -08001369 def IncrementalOTA_InstallBegin(self):
1370 """Called at the start of incremental OTA installation (after
1371 verification is complete)."""
1372 return self._DoCall("IncrementalOTA_InstallBegin")
1373
Doug Zongker05d3dea2009-06-22 11:32:31 -07001374 def IncrementalOTA_InstallEnd(self):
1375 """Called at the end of incremental OTA installation; typically
1376 this is used to install the image for the device's baseband
1377 processor."""
1378 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001379
Tao Bao9bc6bb22015-11-09 16:58:28 -08001380 def VerifyOTA_Assertions(self):
1381 return self._DoCall("VerifyOTA_Assertions")
1382
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001383class File(object):
YOUNG HO CHAccc5c402016-10-13 13:40:46 +09001384 def __init__(self, name, data, compress_size = None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001385 self.name = name
1386 self.data = data
1387 self.size = len(data)
YOUNG HO CHAccc5c402016-10-13 13:40:46 +09001388 self.compress_size = compress_size or self.size
Doug Zongker55d93282011-01-25 17:03:34 -08001389 self.sha1 = sha1(data).hexdigest()
1390
1391 @classmethod
1392 def FromLocalFile(cls, name, diskname):
1393 f = open(diskname, "rb")
1394 data = f.read()
1395 f.close()
1396 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001397
1398 def WriteToTemp(self):
1399 t = tempfile.NamedTemporaryFile()
1400 t.write(self.data)
1401 t.flush()
1402 return t
1403
Dan Willemsen2ee00d52017-03-05 19:51:56 -08001404 def WriteToDir(self, d):
1405 with open(os.path.join(d, self.name), "wb") as fp:
1406 fp.write(self.data)
1407
Geremy Condra36bd3652014-02-06 19:45:10 -08001408 def AddToZip(self, z, compression=None):
Tao Baof3282b42015-04-01 11:21:55 -07001409 ZipWriteStr(z, self.name, self.data, compress_type=compression)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001410
1411DIFF_PROGRAM_BY_EXT = {
1412 ".gz" : "imgdiff",
1413 ".zip" : ["imgdiff", "-z"],
1414 ".jar" : ["imgdiff", "-z"],
1415 ".apk" : ["imgdiff", "-z"],
1416 ".img" : "imgdiff",
1417 }
1418
1419class Difference(object):
Doug Zongker24cd2802012-08-14 16:36:15 -07001420 def __init__(self, tf, sf, diff_program=None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001421 self.tf = tf
1422 self.sf = sf
1423 self.patch = None
Doug Zongker24cd2802012-08-14 16:36:15 -07001424 self.diff_program = diff_program
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001425
1426 def ComputePatch(self):
1427 """Compute the patch (as a string of data) needed to turn sf into
1428 tf. Returns the same tuple as GetPatch()."""
1429
1430 tf = self.tf
1431 sf = self.sf
1432
Doug Zongker24cd2802012-08-14 16:36:15 -07001433 if self.diff_program:
1434 diff_program = self.diff_program
1435 else:
1436 ext = os.path.splitext(tf.name)[1]
1437 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001438
1439 ttemp = tf.WriteToTemp()
1440 stemp = sf.WriteToTemp()
1441
1442 ext = os.path.splitext(tf.name)[1]
1443
1444 try:
1445 ptemp = tempfile.NamedTemporaryFile()
1446 if isinstance(diff_program, list):
1447 cmd = copy.copy(diff_program)
1448 else:
1449 cmd = [diff_program]
1450 cmd.append(stemp.name)
1451 cmd.append(ttemp.name)
1452 cmd.append(ptemp.name)
1453 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Doug Zongkerf8340082014-08-05 10:39:37 -07001454 err = []
1455 def run():
1456 _, e = p.communicate()
Dan Albert8b72aef2015-03-23 19:13:21 -07001457 if e:
1458 err.append(e)
Doug Zongkerf8340082014-08-05 10:39:37 -07001459 th = threading.Thread(target=run)
1460 th.start()
1461 th.join(timeout=300) # 5 mins
1462 if th.is_alive():
Tao Bao89fbb0f2017-01-10 10:47:58 -08001463 print("WARNING: diff command timed out")
Doug Zongkerf8340082014-08-05 10:39:37 -07001464 p.terminate()
1465 th.join(5)
1466 if th.is_alive():
1467 p.kill()
1468 th.join()
1469
Tianjie Xua2a9f992018-01-05 15:15:54 -08001470 if p.returncode != 0:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001471 print("WARNING: failure running %s:\n%s\n" % (
1472 diff_program, "".join(err)))
Doug Zongkerf8340082014-08-05 10:39:37 -07001473 self.patch = None
1474 return None, None, None
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001475 diff = ptemp.read()
1476 finally:
1477 ptemp.close()
1478 stemp.close()
1479 ttemp.close()
1480
1481 self.patch = diff
1482 return self.tf, self.sf, self.patch
1483
1484
1485 def GetPatch(self):
1486 """Return a tuple (target_file, source_file, patch_data).
1487 patch_data may be None if ComputePatch hasn't been called, or if
1488 computing the patch failed."""
1489 return self.tf, self.sf, self.patch
1490
1491
1492def ComputeDifferences(diffs):
1493 """Call ComputePatch on all the Difference objects in 'diffs'."""
Tao Bao89fbb0f2017-01-10 10:47:58 -08001494 print(len(diffs), "diffs to compute")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001495
1496 # Do the largest files first, to try and reduce the long-pole effect.
1497 by_size = [(i.tf.size, i) for i in diffs]
1498 by_size.sort(reverse=True)
1499 by_size = [i[1] for i in by_size]
1500
1501 lock = threading.Lock()
1502 diff_iter = iter(by_size) # accessed under lock
1503
1504 def worker():
1505 try:
1506 lock.acquire()
1507 for d in diff_iter:
1508 lock.release()
1509 start = time.time()
1510 d.ComputePatch()
1511 dur = time.time() - start
1512 lock.acquire()
1513
1514 tf, sf, patch = d.GetPatch()
1515 if sf.name == tf.name:
1516 name = tf.name
1517 else:
1518 name = "%s (%s)" % (tf.name, sf.name)
1519 if patch is None:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001520 print("patching failed! %s" % (name,))
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001521 else:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001522 print("%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
1523 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name))
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001524 lock.release()
Dan Albert8b72aef2015-03-23 19:13:21 -07001525 except Exception as e:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001526 print(e)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001527 raise
1528
1529 # start worker threads; wait for them all to finish.
1530 threads = [threading.Thread(target=worker)
1531 for i in range(OPTIONS.worker_threads)]
1532 for th in threads:
1533 th.start()
1534 while threads:
1535 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -07001536
1537
Dan Albert8b72aef2015-03-23 19:13:21 -07001538class BlockDifference(object):
1539 def __init__(self, partition, tgt, src=None, check_first_block=False,
Tao Bao293fd132016-06-11 12:19:23 -07001540 version=None, disable_imgdiff=False):
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001541 self.tgt = tgt
1542 self.src = src
1543 self.partition = partition
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001544 self.check_first_block = check_first_block
Tao Bao293fd132016-06-11 12:19:23 -07001545 self.disable_imgdiff = disable_imgdiff
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001546
Tao Baodd2a5892015-03-12 12:32:37 -07001547 if version is None:
Tao Bao0582cb62017-12-21 11:47:01 -08001548 version = max(
1549 int(i) for i in
1550 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(","))
Tao Bao8fad03e2017-03-01 14:36:26 -08001551 assert version >= 3
Tao Baodd2a5892015-03-12 12:32:37 -07001552 self.version = version
Doug Zongker62338182014-09-08 08:29:55 -07001553
1554 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads,
Tao Bao293fd132016-06-11 12:19:23 -07001555 version=self.version,
1556 disable_imgdiff=self.disable_imgdiff)
Tao Bao04bce3a2018-02-28 11:11:00 -08001557 self.path = os.path.join(MakeTempDir(), partition)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001558 b.Compute(self.path)
Tao Baod8d14be2016-02-04 14:26:02 -08001559 self._required_cache = b.max_stashed_size
Tao Baod522bdc2016-04-12 15:53:16 -07001560 self.touched_src_ranges = b.touched_src_ranges
1561 self.touched_src_sha1 = b.touched_src_sha1
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001562
Tao Baoaac4ad52015-10-16 15:26:34 -07001563 if src is None:
1564 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)
1565 else:
1566 _, self.device = GetTypeAndDevice("/" + partition,
1567 OPTIONS.source_info_dict)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001568
Tao Baod8d14be2016-02-04 14:26:02 -08001569 @property
1570 def required_cache(self):
1571 return self._required_cache
1572
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001573 def WriteScript(self, script, output_zip, progress=None):
1574 if not self.src:
1575 # write the output unconditionally
Jesse Zhao75bcea02015-01-06 10:59:53 -08001576 script.Print("Patching %s image unconditionally..." % (self.partition,))
1577 else:
1578 script.Print("Patching %s image after verification." % (self.partition,))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001579
Dan Albert8b72aef2015-03-23 19:13:21 -07001580 if progress:
1581 script.ShowProgress(progress, 0)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001582 self._WriteUpdate(script, output_zip)
Tianjie Xub2deb222016-03-25 15:01:33 -07001583 if OPTIONS.verify:
1584 self._WritePostInstallVerifyScript(script)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001585
Tao Bao9bc6bb22015-11-09 16:58:28 -08001586 def WriteStrictVerifyScript(self, script):
1587 """Verify all the blocks in the care_map, including clobbered blocks.
1588
1589 This differs from the WriteVerifyScript() function: a) it prints different
1590 error messages; b) it doesn't allow half-way updated images to pass the
1591 verification."""
1592
1593 partition = self.partition
1594 script.Print("Verifying %s..." % (partition,))
1595 ranges = self.tgt.care_map
1596 ranges_str = ranges.to_string_raw()
1597 script.AppendExtra('range_sha1("%s", "%s") == "%s" && '
1598 'ui_print(" Verified.") || '
1599 'ui_print("\\"%s\\" has unexpected contents.");' % (
1600 self.device, ranges_str,
1601 self.tgt.TotalSha1(include_clobbered_blocks=True),
1602 self.device))
1603 script.AppendExtra("")
1604
Tao Baod522bdc2016-04-12 15:53:16 -07001605 def WriteVerifyScript(self, script, touched_blocks_only=False):
Sami Tolvanendd67a292014-12-09 16:40:34 +00001606 partition = self.partition
Tao Baof9efe282016-04-14 15:58:05 -07001607
1608 # full OTA
Jesse Zhao75bcea02015-01-06 10:59:53 -08001609 if not self.src:
Sami Tolvanendd67a292014-12-09 16:40:34 +00001610 script.Print("Image %s will be patched unconditionally." % (partition,))
Tao Baof9efe282016-04-14 15:58:05 -07001611
1612 # incremental OTA
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001613 else:
Tao Bao8fad03e2017-03-01 14:36:26 -08001614 if touched_blocks_only:
Tao Baod522bdc2016-04-12 15:53:16 -07001615 ranges = self.touched_src_ranges
1616 expected_sha1 = self.touched_src_sha1
1617 else:
1618 ranges = self.src.care_map.subtract(self.src.clobbered_blocks)
1619 expected_sha1 = self.src.TotalSha1()
Tao Baof9efe282016-04-14 15:58:05 -07001620
1621 # No blocks to be checked, skipping.
1622 if not ranges:
1623 return
1624
Tao Bao5ece99d2015-05-12 11:42:31 -07001625 ranges_str = ranges.to_string_raw()
Tao Bao8fad03e2017-03-01 14:36:26 -08001626 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || '
1627 'block_image_verify("%s", '
1628 'package_extract_file("%s.transfer.list"), '
1629 '"%s.new.dat", "%s.patch.dat")) then') % (
1630 self.device, ranges_str, expected_sha1,
1631 self.device, partition, partition, partition))
Tao Baodd2a5892015-03-12 12:32:37 -07001632 script.Print('Verified %s image...' % (partition,))
Dan Albert8b72aef2015-03-23 19:13:21 -07001633 script.AppendExtra('else')
Sami Tolvanendd67a292014-12-09 16:40:34 +00001634
Tianjie Xufc3422a2015-12-15 11:53:59 -08001635 if self.version >= 4:
1636
1637 # Bug: 21124327
1638 # When generating incrementals for the system and vendor partitions in
1639 # version 4 or newer, explicitly check the first block (which contains
1640 # the superblock) of the partition to see if it's what we expect. If
1641 # this check fails, give an explicit log message about the partition
1642 # having been remounted R/W (the most likely explanation).
1643 if self.check_first_block:
1644 script.AppendExtra('check_first_block("%s");' % (self.device,))
1645
1646 # If version >= 4, try block recovery before abort update
Tianjie Xu209db462016-05-24 17:34:52 -07001647 if partition == "system":
1648 code = ErrorCode.SYSTEM_RECOVER_FAILURE
1649 else:
1650 code = ErrorCode.VENDOR_RECOVER_FAILURE
Tianjie Xufc3422a2015-12-15 11:53:59 -08001651 script.AppendExtra((
1652 'ifelse (block_image_recover("{device}", "{ranges}") && '
1653 'block_image_verify("{device}", '
1654 'package_extract_file("{partition}.transfer.list"), '
1655 '"{partition}.new.dat", "{partition}.patch.dat"), '
1656 'ui_print("{partition} recovered successfully."), '
Tianjie Xu209db462016-05-24 17:34:52 -07001657 'abort("E{code}: {partition} partition fails to recover"));\n'
Tianjie Xufc3422a2015-12-15 11:53:59 -08001658 'endif;').format(device=self.device, ranges=ranges_str,
Tianjie Xu209db462016-05-24 17:34:52 -07001659 partition=partition, code=code))
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001660
Tao Baodd2a5892015-03-12 12:32:37 -07001661 # Abort the OTA update. Note that the incremental OTA cannot be applied
1662 # even if it may match the checksum of the target partition.
1663 # a) If version < 3, operations like move and erase will make changes
1664 # unconditionally and damage the partition.
1665 # b) If version >= 3, it won't even reach here.
Tianjie Xufc3422a2015-12-15 11:53:59 -08001666 else:
Tianjie Xu209db462016-05-24 17:34:52 -07001667 if partition == "system":
1668 code = ErrorCode.SYSTEM_VERIFICATION_FAILURE
1669 else:
1670 code = ErrorCode.VENDOR_VERIFICATION_FAILURE
1671 script.AppendExtra((
1672 'abort("E%d: %s partition has unexpected contents");\n'
1673 'endif;') % (code, partition))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001674
Tao Bao5fcaaef2015-06-01 13:40:49 -07001675 def _WritePostInstallVerifyScript(self, script):
1676 partition = self.partition
1677 script.Print('Verifying the updated %s image...' % (partition,))
1678 # Unlike pre-install verification, clobbered_blocks should not be ignored.
1679 ranges = self.tgt.care_map
1680 ranges_str = ranges.to_string_raw()
1681 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
1682 self.device, ranges_str,
1683 self.tgt.TotalSha1(include_clobbered_blocks=True)))
Tao Baoe9b61912015-07-09 17:37:49 -07001684
1685 # Bug: 20881595
1686 # Verify that extended blocks are really zeroed out.
1687 if self.tgt.extended:
1688 ranges_str = self.tgt.extended.to_string_raw()
1689 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
1690 self.device, ranges_str,
1691 self._HashZeroBlocks(self.tgt.extended.size())))
1692 script.Print('Verified the updated %s image.' % (partition,))
Tianjie Xu209db462016-05-24 17:34:52 -07001693 if partition == "system":
1694 code = ErrorCode.SYSTEM_NONZERO_CONTENTS
1695 else:
1696 code = ErrorCode.VENDOR_NONZERO_CONTENTS
Tao Baoe9b61912015-07-09 17:37:49 -07001697 script.AppendExtra(
1698 'else\n'
Tianjie Xu209db462016-05-24 17:34:52 -07001699 ' abort("E%d: %s partition has unexpected non-zero contents after '
1700 'OTA update");\n'
1701 'endif;' % (code, partition))
Tao Baoe9b61912015-07-09 17:37:49 -07001702 else:
1703 script.Print('Verified the updated %s image.' % (partition,))
1704
Tianjie Xu209db462016-05-24 17:34:52 -07001705 if partition == "system":
1706 code = ErrorCode.SYSTEM_UNEXPECTED_CONTENTS
1707 else:
1708 code = ErrorCode.VENDOR_UNEXPECTED_CONTENTS
1709
Tao Bao5fcaaef2015-06-01 13:40:49 -07001710 script.AppendExtra(
1711 'else\n'
Tianjie Xu209db462016-05-24 17:34:52 -07001712 ' abort("E%d: %s partition has unexpected contents after OTA '
1713 'update");\n'
1714 'endif;' % (code, partition))
Tao Bao5fcaaef2015-06-01 13:40:49 -07001715
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001716 def _WriteUpdate(self, script, output_zip):
Dan Albert8e0178d2015-01-27 15:53:15 -08001717 ZipWrite(output_zip,
1718 '{}.transfer.list'.format(self.path),
1719 '{}.transfer.list'.format(self.partition))
Tianjie Xub0a29ad2017-07-06 15:13:59 -07001720
1721 # For full OTA, compress the new.dat with brotli with quality 6 to reduce its size. Quailty 9
1722 # almost triples the compression time but doesn't further reduce the size too much.
1723 # For a typical 1.8G system.new.dat
1724 # zip | brotli(quality 6) | brotli(quality 9)
1725 # compressed_size: 942M | 869M (~8% reduced) | 854M
1726 # compression_time: 75s | 265s | 719s
1727 # decompression_time: 15s | 25s | 25s
1728
1729 if not self.src:
Alex Deymob10e07a2017-11-09 23:53:42 +01001730 brotli_cmd = ['brotli', '--quality=6',
1731 '--output={}.new.dat.br'.format(self.path),
1732 '{}.new.dat'.format(self.path)]
Tianjie Xub0a29ad2017-07-06 15:13:59 -07001733 print("Compressing {}.new.dat with brotli".format(self.partition))
Alex Deymob10e07a2017-11-09 23:53:42 +01001734 p = Run(brotli_cmd, stdout=subprocess.PIPE)
Tianjie Xub0a29ad2017-07-06 15:13:59 -07001735 p.communicate()
1736 assert p.returncode == 0,\
1737 'compression of {}.new.dat failed'.format(self.partition)
1738
1739 new_data_name = '{}.new.dat.br'.format(self.partition)
1740 ZipWrite(output_zip,
1741 '{}.new.dat.br'.format(self.path),
1742 new_data_name,
1743 compress_type=zipfile.ZIP_STORED)
1744 else:
1745 new_data_name = '{}.new.dat'.format(self.partition)
1746 ZipWrite(output_zip, '{}.new.dat'.format(self.path), new_data_name)
1747
Dan Albert8e0178d2015-01-27 15:53:15 -08001748 ZipWrite(output_zip,
1749 '{}.patch.dat'.format(self.path),
1750 '{}.patch.dat'.format(self.partition),
1751 compress_type=zipfile.ZIP_STORED)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001752
Tianjie Xu209db462016-05-24 17:34:52 -07001753 if self.partition == "system":
1754 code = ErrorCode.SYSTEM_UPDATE_FAILURE
1755 else:
1756 code = ErrorCode.VENDOR_UPDATE_FAILURE
1757
Dan Albert8e0178d2015-01-27 15:53:15 -08001758 call = ('block_image_update("{device}", '
1759 'package_extract_file("{partition}.transfer.list"), '
Tianjie Xub0a29ad2017-07-06 15:13:59 -07001760 '"{new_data_name}", "{partition}.patch.dat") ||\n'
Tianjie Xu209db462016-05-24 17:34:52 -07001761 ' abort("E{code}: Failed to update {partition} image.");'.format(
Tianjie Xub0a29ad2017-07-06 15:13:59 -07001762 device=self.device, partition=self.partition,
1763 new_data_name=new_data_name, code=code))
Dan Albert8b72aef2015-03-23 19:13:21 -07001764 script.AppendExtra(script.WordWrap(call))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001765
Dan Albert8b72aef2015-03-23 19:13:21 -07001766 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use
Sami Tolvanendd67a292014-12-09 16:40:34 +00001767 data = source.ReadRangeSet(ranges)
1768 ctx = sha1()
1769
1770 for p in data:
1771 ctx.update(p)
1772
1773 return ctx.hexdigest()
1774
Tao Baoe9b61912015-07-09 17:37:49 -07001775 def _HashZeroBlocks(self, num_blocks): # pylint: disable=no-self-use
1776 """Return the hash value for all zero blocks."""
1777 zero_block = '\x00' * 4096
1778 ctx = sha1()
1779 for _ in range(num_blocks):
1780 ctx.update(zero_block)
1781
1782 return ctx.hexdigest()
1783
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001784
1785DataImage = blockimgdiff.DataImage
1786
Doug Zongker96a57e72010-09-26 14:57:41 -07001787# map recovery.fstab's fs_types to mount/format "partition types"
Dan Albert8b72aef2015-03-23 19:13:21 -07001788PARTITION_TYPES = {
Dan Albert8b72aef2015-03-23 19:13:21 -07001789 "ext4": "EMMC",
1790 "emmc": "EMMC",
Mohamad Ayyash95e74c12015-05-01 15:39:36 -07001791 "f2fs": "EMMC",
Brandon Bennett6287e2b2011-11-19 16:02:04 -07001792 "squashfs": "EMMC",
1793 "ext2": "EMMC",
1794 "ext3": "EMMC",
1795 "vfat": "EMMC"
Dan Albert8b72aef2015-03-23 19:13:21 -07001796}
Doug Zongker96a57e72010-09-26 14:57:41 -07001797
1798def GetTypeAndDevice(mount_point, info):
1799 fstab = info["fstab"]
1800 if fstab:
Dan Albert8b72aef2015-03-23 19:13:21 -07001801 return (PARTITION_TYPES[fstab[mount_point].fs_type],
1802 fstab[mount_point].device)
Doug Zongker96a57e72010-09-26 14:57:41 -07001803 else:
Dan Albert8b72aef2015-03-23 19:13:21 -07001804 raise KeyError
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00001805
1806
1807def ParseCertificate(data):
Tao Bao17e4e612018-02-16 17:12:54 -08001808 """Parses and converts a PEM-encoded certificate into DER-encoded.
1809
1810 This gives the same result as `openssl x509 -in <filename> -outform DER`.
1811
1812 Returns:
1813 The decoded certificate string.
1814 """
1815 cert_buffer = []
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00001816 save = False
1817 for line in data.split("\n"):
1818 if "--END CERTIFICATE--" in line:
1819 break
1820 if save:
Tao Bao17e4e612018-02-16 17:12:54 -08001821 cert_buffer.append(line)
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00001822 if "--BEGIN CERTIFICATE--" in line:
1823 save = True
Tao Bao17e4e612018-02-16 17:12:54 -08001824 cert = "".join(cert_buffer).decode('base64')
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00001825 return cert
Doug Zongkerc9253822014-02-04 12:17:58 -08001826
Tao Bao04e1f012018-02-04 12:13:35 -08001827
1828def ExtractPublicKey(cert):
1829 """Extracts the public key (PEM-encoded) from the given certificate file.
1830
1831 Args:
1832 cert: The certificate filename.
1833
1834 Returns:
1835 The public key string.
1836
1837 Raises:
1838 AssertionError: On non-zero return from 'openssl'.
1839 """
1840 # The behavior with '-out' is different between openssl 1.1 and openssl 1.0.
1841 # While openssl 1.1 writes the key into the given filename followed by '-out',
1842 # openssl 1.0 (both of 1.0.1 and 1.0.2) doesn't. So we collect the output from
1843 # stdout instead.
1844 cmd = ['openssl', 'x509', '-pubkey', '-noout', '-in', cert]
1845 proc = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1846 pubkey, stderrdata = proc.communicate()
1847 assert proc.returncode == 0, \
1848 'Failed to dump public key from certificate: %s\n%s' % (cert, stderrdata)
1849 return pubkey
1850
1851
Doug Zongker412c02f2014-02-13 10:58:24 -08001852def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img,
1853 info_dict=None):
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001854 """Generates the recovery-from-boot patch and writes the script to output.
Doug Zongkerc9253822014-02-04 12:17:58 -08001855
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001856 Most of the space in the boot and recovery images is just the kernel, which is
1857 identical for the two, so the resulting patch should be efficient. Add it to
1858 the output zip, along with a shell script that is run from init.rc on first
1859 boot to actually do the patching and install the new recovery image.
1860
1861 Args:
1862 input_dir: The top-level input directory of the target-files.zip.
1863 output_sink: The callback function that writes the result.
1864 recovery_img: File object for the recovery image.
1865 boot_img: File objects for the boot image.
1866 info_dict: A dict returned by common.LoadInfoDict() on the input
1867 target_files. Will use OPTIONS.info_dict if None has been given.
Doug Zongkerc9253822014-02-04 12:17:58 -08001868 """
Doug Zongker412c02f2014-02-13 10:58:24 -08001869 if info_dict is None:
1870 info_dict = OPTIONS.info_dict
1871
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001872 full_recovery_image = info_dict.get("full_recovery_image") == "true"
Dan Pasanenb380dc82016-12-01 20:06:14 -06001873 use_bsdiff = info_dict.get("no_gzip_recovery_ramdisk") == "true"
Doug Zongkerc9253822014-02-04 12:17:58 -08001874
Tao Baof2cffbd2015-07-22 12:33:18 -07001875 if full_recovery_image:
1876 output_sink("etc/recovery.img", recovery_img.data)
1877
1878 else:
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001879 system_root_image = info_dict.get("system_root_image") == "true"
Tao Baof2cffbd2015-07-22 12:33:18 -07001880 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat")
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001881 # With system-root-image, boot and recovery images will have mismatching
1882 # entries (only recovery has the ramdisk entry) (Bug: 72731506). Use bsdiff
1883 # to handle such a case.
Dan Pasanenb380dc82016-12-01 20:06:14 -06001884 if system_root_image or use_bsdiff:
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001885 diff_program = ["bsdiff"]
Tao Baof2cffbd2015-07-22 12:33:18 -07001886 bonus_args = ""
Tao Bao2d2ce7d2018-03-09 17:04:42 -08001887 assert not os.path.exists(path)
1888 else:
1889 diff_program = ["imgdiff"]
1890 if os.path.exists(path):
1891 diff_program.append("-b")
1892 diff_program.append(path)
1893 bonus_args = "-b /system/etc/recovery-resource.dat"
1894 else:
1895 bonus_args = ""
Tao Baof2cffbd2015-07-22 12:33:18 -07001896
1897 d = Difference(recovery_img, boot_img, diff_program=diff_program)
1898 _, _, patch = d.ComputePatch()
1899 output_sink("recovery-from-boot.p", patch)
Doug Zongkerc9253822014-02-04 12:17:58 -08001900
Dan Albertebb19aa2015-03-27 19:11:53 -07001901 try:
Tao Bao6f0b2192015-10-13 16:37:12 -07001902 # The following GetTypeAndDevice()s need to use the path in the target
1903 # info_dict instead of source_info_dict.
Dan Albertebb19aa2015-03-27 19:11:53 -07001904 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict)
1905 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict)
1906 except KeyError:
Ying Wanga961a092014-07-29 11:42:37 -07001907 return
Doug Zongkerc9253822014-02-04 12:17:58 -08001908
Tao Baof2cffbd2015-07-22 12:33:18 -07001909 if full_recovery_image:
1910 sh = """#!/system/bin/sh
1911if ! applypatch -c %(type)s:%(device)s:%(size)d:%(sha1)s; then
1912 applypatch /system/etc/recovery.img %(type)s:%(device)s %(sha1)s %(size)d && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed"
1913else
1914 log -t recovery "Recovery image already installed"
1915fi
1916""" % {'type': recovery_type,
1917 'device': recovery_device,
1918 'sha1': recovery_img.sha1,
1919 'size': recovery_img.size}
1920 else:
1921 sh = """#!/system/bin/sh
Doug Zongkerc9253822014-02-04 12:17:58 -08001922if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then
1923 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed"
1924else
1925 log -t recovery "Recovery image already installed"
1926fi
Dan Albert8b72aef2015-03-23 19:13:21 -07001927""" % {'boot_size': boot_img.size,
1928 'boot_sha1': boot_img.sha1,
1929 'recovery_size': recovery_img.size,
1930 'recovery_sha1': recovery_img.sha1,
1931 'boot_type': boot_type,
1932 'boot_device': boot_device,
1933 'recovery_type': recovery_type,
1934 'recovery_device': recovery_device,
1935 'bonus_args': bonus_args}
Doug Zongkerc9253822014-02-04 12:17:58 -08001936
1937 # The install script location moved from /system/etc to /system/bin
Tianjie Xu78de9f12017-06-20 16:52:54 -07001938 # in the L release.
1939 sh_location = "bin/install-recovery.sh"
Tao Bao9f0c8df2015-07-07 18:31:47 -07001940
Tao Bao89fbb0f2017-01-10 10:47:58 -08001941 print("putting script in", sh_location)
Doug Zongkerc9253822014-02-04 12:17:58 -08001942
1943 output_sink(sh_location, sh)