blob: 34c13596c73faa0bf05fd0cadf0614309c5d96a4 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Tao Bao89fbb0f2017-01-10 10:47:58 -080015from __future__ import print_function
16
Yifan Hong10c530d2018-12-27 17:34:18 -080017import collections
Doug Zongkerea5d7a92010-09-12 15:26:16 -070018import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070019import errno
Tao Bao0ff15de2019-03-20 11:26:06 -070020import fnmatch
Doug Zongkereef39442009-04-02 12:14:19 -070021import getopt
22import getpass
Narayan Kamatha07bf042017-08-14 14:49:21 +010023import gzip
Doug Zongker05d3dea2009-06-22 11:32:31 -070024import imp
Tao Bao32fcdab2018-10-12 10:30:39 -070025import json
26import logging
27import logging.config
Doug Zongkereef39442009-04-02 12:14:19 -070028import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080029import platform
Doug Zongkereef39442009-04-02 12:14:19 -070030import re
T.R. Fullhart37e10522013-03-18 10:31:26 -070031import shlex
Doug Zongkereef39442009-04-02 12:14:19 -070032import shutil
Tao Baoc765cca2018-01-31 17:32:40 -080033import string
Doug Zongkereef39442009-04-02 12:14:19 -070034import subprocess
35import sys
36import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070037import threading
38import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070039import zipfile
Tao Bao12d87fc2018-01-31 12:18:52 -080040from hashlib import sha1, sha256
Doug Zongkereef39442009-04-02 12:14:19 -070041
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070042import blockimgdiff
Tao Baoc765cca2018-01-31 17:32:40 -080043import sparse_img
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070044
Tao Bao32fcdab2018-10-12 10:30:39 -070045logger = logging.getLogger(__name__)
46
Tao Bao986ee862018-10-04 15:46:16 -070047
Dan Albert8b72aef2015-03-23 19:13:21 -070048class Options(object):
49 def __init__(self):
Pavel Salomatov32676552019-03-06 20:00:45 +030050 base_out_path = os.getenv('OUT_DIR_COMMON_BASE')
51 if base_out_path is None:
52 base_search_path = "out"
53 else:
Tao Bao2cc0ca12019-03-15 10:44:43 -070054 base_search_path = os.path.join(base_out_path,
55 os.path.basename(os.getcwd()))
Pavel Salomatov32676552019-03-06 20:00:45 +030056
Dan Albert8b72aef2015-03-23 19:13:21 -070057 platform_search_path = {
Pavel Salomatov32676552019-03-06 20:00:45 +030058 "linux2": os.path.join(base_search_path, "host/linux-x86"),
59 "darwin": os.path.join(base_search_path, "host/darwin-x86"),
Doug Zongker85448772014-09-09 14:59:20 -070060 }
Doug Zongker85448772014-09-09 14:59:20 -070061
Tao Bao76def242017-11-21 09:25:31 -080062 self.search_path = platform_search_path.get(sys.platform)
Dan Albert8b72aef2015-03-23 19:13:21 -070063 self.signapk_path = "framework/signapk.jar" # Relative to search_path
Alex Klyubin9667b182015-12-10 13:38:50 -080064 self.signapk_shared_library_path = "lib64" # Relative to search_path
Dan Albert8b72aef2015-03-23 19:13:21 -070065 self.extra_signapk_args = []
66 self.java_path = "java" # Use the one on the path by default.
Tao Baoe95540e2016-11-08 12:08:53 -080067 self.java_args = ["-Xmx2048m"] # The default JVM args.
Dan Albert8b72aef2015-03-23 19:13:21 -070068 self.public_key_suffix = ".x509.pem"
69 self.private_key_suffix = ".pk8"
Dan Albertcd9ecc02015-03-27 16:37:23 -070070 # use otatools built boot_signer by default
71 self.boot_signer_path = "boot_signer"
Baligh Uddin601ddea2015-06-09 15:48:14 -070072 self.boot_signer_args = []
73 self.verity_signer_path = None
74 self.verity_signer_args = []
Dan Albert8b72aef2015-03-23 19:13:21 -070075 self.verbose = False
76 self.tempfiles = []
77 self.device_specific = None
78 self.extras = {}
79 self.info_dict = None
Tao Bao6f0b2192015-10-13 16:37:12 -070080 self.source_info_dict = None
81 self.target_info_dict = None
Dan Albert8b72aef2015-03-23 19:13:21 -070082 self.worker_threads = None
Tao Bao575d68a2015-08-07 19:49:45 -070083 # Stash size cannot exceed cache_size * threshold.
84 self.cache_size = None
85 self.stash_threshold = 0.8
Dan Albert8b72aef2015-03-23 19:13:21 -070086
87
88OPTIONS = Options()
Doug Zongkereef39442009-04-02 12:14:19 -070089
Tao Bao71197512018-10-11 14:08:45 -070090# The block size that's used across the releasetools scripts.
91BLOCK_SIZE = 4096
92
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080093# Values for "certificate" in apkcerts that mean special things.
94SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
95
Tao Bao9dd909e2017-11-14 11:27:32 -080096# The partitions allowed to be signed by AVB (Android verified boot 2.0).
Dario Freni5f681e12018-05-29 13:09:01 +010097AVB_PARTITIONS = ('boot', 'recovery', 'system', 'vendor', 'product',
Dario Freni924af7d2018-08-17 00:56:14 +010098 'product_services', 'dtbo', 'odm')
Tao Bao9dd909e2017-11-14 11:27:32 -080099
Tianjie Xu861f4132018-09-12 11:49:33 -0700100# Partitions that should have their care_map added to META/care_map.pb
101PARTITIONS_WITH_CARE_MAP = ('system', 'vendor', 'product', 'product_services',
102 'odm')
103
104
Tianjie Xu209db462016-05-24 17:34:52 -0700105class ErrorCode(object):
106 """Define error_codes for failures that happen during the actual
107 update package installation.
108
109 Error codes 0-999 are reserved for failures before the package
110 installation (i.e. low battery, package verification failure).
111 Detailed code in 'bootable/recovery/error_code.h' """
112
113 SYSTEM_VERIFICATION_FAILURE = 1000
114 SYSTEM_UPDATE_FAILURE = 1001
115 SYSTEM_UNEXPECTED_CONTENTS = 1002
116 SYSTEM_NONZERO_CONTENTS = 1003
117 SYSTEM_RECOVER_FAILURE = 1004
118 VENDOR_VERIFICATION_FAILURE = 2000
119 VENDOR_UPDATE_FAILURE = 2001
120 VENDOR_UNEXPECTED_CONTENTS = 2002
121 VENDOR_NONZERO_CONTENTS = 2003
122 VENDOR_RECOVER_FAILURE = 2004
123 OEM_PROP_MISMATCH = 3000
124 FINGERPRINT_MISMATCH = 3001
125 THUMBPRINT_MISMATCH = 3002
126 OLDER_BUILD = 3003
127 DEVICE_MISMATCH = 3004
128 BAD_PATCH_FILE = 3005
129 INSUFFICIENT_CACHE_SPACE = 3006
130 TUNE_PARTITION_FAILURE = 3007
131 APPLY_PATCH_FAILURE = 3008
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800132
Tao Bao80921982018-03-21 21:02:19 -0700133
Dan Albert8b72aef2015-03-23 19:13:21 -0700134class ExternalError(RuntimeError):
135 pass
Doug Zongkereef39442009-04-02 12:14:19 -0700136
137
Tao Bao32fcdab2018-10-12 10:30:39 -0700138def InitLogging():
139 DEFAULT_LOGGING_CONFIG = {
140 'version': 1,
141 'disable_existing_loggers': False,
142 'formatters': {
143 'standard': {
144 'format':
145 '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s',
146 'datefmt': '%Y-%m-%d %H:%M:%S',
147 },
148 },
149 'handlers': {
150 'default': {
151 'class': 'logging.StreamHandler',
152 'formatter': 'standard',
153 },
154 },
155 'loggers': {
156 '': {
157 'handlers': ['default'],
158 'level': 'WARNING',
159 'propagate': True,
160 }
161 }
162 }
163 env_config = os.getenv('LOGGING_CONFIG')
164 if env_config:
165 with open(env_config) as f:
166 config = json.load(f)
167 else:
168 config = DEFAULT_LOGGING_CONFIG
169
170 # Increase the logging level for verbose mode.
171 if OPTIONS.verbose:
172 config = copy.deepcopy(DEFAULT_LOGGING_CONFIG)
173 config['loggers']['']['level'] = 'INFO'
174
175 logging.config.dictConfig(config)
176
177
Tao Bao39451582017-05-04 11:10:47 -0700178def Run(args, verbose=None, **kwargs):
Tao Bao73dd4f42018-10-04 16:25:33 -0700179 """Creates and returns a subprocess.Popen object.
Tao Bao39451582017-05-04 11:10:47 -0700180
Tao Bao73dd4f42018-10-04 16:25:33 -0700181 Args:
182 args: The command represented as a list of strings.
Tao Bao32fcdab2018-10-12 10:30:39 -0700183 verbose: Whether the commands should be shown. Default to the global
184 verbosity if unspecified.
Tao Bao73dd4f42018-10-04 16:25:33 -0700185 kwargs: Any additional args to be passed to subprocess.Popen(), such as env,
186 stdin, etc. stdout and stderr will default to subprocess.PIPE and
187 subprocess.STDOUT respectively unless caller specifies any of them.
188
189 Returns:
190 A subprocess.Popen object.
Tao Bao39451582017-05-04 11:10:47 -0700191 """
Tao Bao73dd4f42018-10-04 16:25:33 -0700192 if 'stdout' not in kwargs and 'stderr' not in kwargs:
193 kwargs['stdout'] = subprocess.PIPE
194 kwargs['stderr'] = subprocess.STDOUT
Tao Bao32fcdab2018-10-12 10:30:39 -0700195 # Don't log any if caller explicitly says so.
196 if verbose != False:
197 logger.info(" Running: \"%s\"", " ".join(args))
Doug Zongkereef39442009-04-02 12:14:19 -0700198 return subprocess.Popen(args, **kwargs)
199
200
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800201def RunAndWait(args, verbose=None, **kwargs):
Bill Peckham889b0c62019-02-21 18:53:37 -0800202 """Runs the given command waiting for it to complete.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800203
204 Args:
205 args: The command represented as a list of strings.
206 verbose: Whether the commands should be shown. Default to the global
207 verbosity if unspecified.
208 kwargs: Any additional args to be passed to subprocess.Popen(), such as env,
209 stdin, etc. stdout and stderr will default to subprocess.PIPE and
210 subprocess.STDOUT respectively unless caller specifies any of them.
211
Bill Peckham889b0c62019-02-21 18:53:37 -0800212 Raises:
213 ExternalError: On non-zero exit from the command.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800214 """
215 proc = Run(args, verbose=verbose, **kwargs)
216 proc.wait()
Bill Peckham889b0c62019-02-21 18:53:37 -0800217
218 if proc.returncode != 0:
219 raise ExternalError(
220 "Failed to run command '{}' (exit code {})".format(
221 args, proc.returncode))
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800222
223
Tao Bao986ee862018-10-04 15:46:16 -0700224def RunAndCheckOutput(args, verbose=None, **kwargs):
225 """Runs the given command and returns the output.
226
227 Args:
228 args: The command represented as a list of strings.
Tao Bao32fcdab2018-10-12 10:30:39 -0700229 verbose: Whether the commands should be shown. Default to the global
230 verbosity if unspecified.
Tao Bao986ee862018-10-04 15:46:16 -0700231 kwargs: Any additional args to be passed to subprocess.Popen(), such as env,
232 stdin, etc. stdout and stderr will default to subprocess.PIPE and
233 subprocess.STDOUT respectively unless caller specifies any of them.
234
235 Returns:
236 The output string.
237
238 Raises:
239 ExternalError: On non-zero exit from the command.
240 """
Tao Bao986ee862018-10-04 15:46:16 -0700241 proc = Run(args, verbose=verbose, **kwargs)
242 output, _ = proc.communicate()
Tao Bao32fcdab2018-10-12 10:30:39 -0700243 # Don't log any if caller explicitly says so.
244 if verbose != False:
245 logger.info("%s", output.rstrip())
Tao Bao986ee862018-10-04 15:46:16 -0700246 if proc.returncode != 0:
247 raise ExternalError(
248 "Failed to run command '{}' (exit code {}):\n{}".format(
249 args, proc.returncode, output))
250 return output
251
252
Tao Baoc765cca2018-01-31 17:32:40 -0800253def RoundUpTo4K(value):
254 rounded_up = value + 4095
255 return rounded_up - (rounded_up % 4096)
256
257
Ying Wang7e6d4e42010-12-13 16:25:36 -0800258def CloseInheritedPipes():
259 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
260 before doing other work."""
261 if platform.system() != "Darwin":
262 return
263 for d in range(3, 1025):
264 try:
265 stat = os.fstat(d)
266 if stat is not None:
267 pipebit = stat[0] & 0x1000
268 if pipebit != 0:
269 os.close(d)
270 except OSError:
271 pass
272
273
Tao Bao410ad8b2018-08-24 12:08:38 -0700274def LoadInfoDict(input_file, repacking=False):
275 """Loads the key/value pairs from the given input target_files.
276
277 It reads `META/misc_info.txt` file in the target_files input, does sanity
278 checks and returns the parsed key/value pairs for to the given build. It's
279 usually called early when working on input target_files files, e.g. when
280 generating OTAs, or signing builds. Note that the function may be called
281 against an old target_files file (i.e. from past dessert releases). So the
282 property parsing needs to be backward compatible.
283
284 In a `META/misc_info.txt`, a few properties are stored as links to the files
285 in the PRODUCT_OUT directory. It works fine with the build system. However,
286 they are no longer available when (re)generating images from target_files zip.
287 When `repacking` is True, redirect these properties to the actual files in the
288 unzipped directory.
289
290 Args:
291 input_file: The input target_files file, which could be an open
292 zipfile.ZipFile instance, or a str for the dir that contains the files
293 unzipped from a target_files file.
294 repacking: Whether it's trying repack an target_files file after loading the
295 info dict (default: False). If so, it will rewrite a few loaded
296 properties (e.g. selinux_fc, root_dir) to point to the actual files in
297 target_files file. When doing repacking, `input_file` must be a dir.
298
299 Returns:
300 A dict that contains the parsed key/value pairs.
301
302 Raises:
303 AssertionError: On invalid input arguments.
304 ValueError: On malformed input values.
305 """
306 if repacking:
307 assert isinstance(input_file, str), \
308 "input_file must be a path str when doing repacking"
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700309
Doug Zongkerc9253822014-02-04 12:17:58 -0800310 def read_helper(fn):
Dan Albert8b72aef2015-03-23 19:13:21 -0700311 if isinstance(input_file, zipfile.ZipFile):
312 return input_file.read(fn)
Doug Zongkerc9253822014-02-04 12:17:58 -0800313 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700314 path = os.path.join(input_file, *fn.split("/"))
Doug Zongkerc9253822014-02-04 12:17:58 -0800315 try:
316 with open(path) as f:
317 return f.read()
Dan Albert8b72aef2015-03-23 19:13:21 -0700318 except IOError as e:
Doug Zongkerc9253822014-02-04 12:17:58 -0800319 if e.errno == errno.ENOENT:
320 raise KeyError(fn)
Tao Bao6cd54732017-02-27 15:12:05 -0800321
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700322 try:
Michael Runge6e836112014-04-15 17:40:21 -0700323 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n"))
Doug Zongker37974732010-09-16 17:44:38 -0700324 except KeyError:
Tao Bao410ad8b2018-08-24 12:08:38 -0700325 raise ValueError("Failed to find META/misc_info.txt in input target-files")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700326
Tao Bao410ad8b2018-08-24 12:08:38 -0700327 if "recovery_api_version" not in d:
328 raise ValueError("Failed to find 'recovery_api_version'")
329 if "fstab_version" not in d:
330 raise ValueError("Failed to find 'fstab_version'")
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800331
Tao Bao410ad8b2018-08-24 12:08:38 -0700332 if repacking:
333 # We carry a copy of file_contexts.bin under META/. If not available, search
334 # BOOT/RAMDISK/. Note that sometimes we may need a different file to build
335 # images than the one running on device, in that case, we must have the one
336 # for image generation copied to META/.
Tao Bao79735a62015-08-28 10:52:03 -0700337 fc_basename = os.path.basename(d.get("selinux_fc", "file_contexts"))
Tao Bao410ad8b2018-08-24 12:08:38 -0700338 fc_config = os.path.join(input_file, "META", fc_basename)
Tom Cherryd14b8952018-08-09 14:26:00 -0700339 assert os.path.exists(fc_config)
Tao Bao2c15d9e2015-07-09 11:51:16 -0700340
Tom Cherryd14b8952018-08-09 14:26:00 -0700341 d["selinux_fc"] = fc_config
Tao Bao2c15d9e2015-07-09 11:51:16 -0700342
Tom Cherryd14b8952018-08-09 14:26:00 -0700343 # Similarly we need to redirect "root_dir", and "root_fs_config".
Tao Bao410ad8b2018-08-24 12:08:38 -0700344 d["root_dir"] = os.path.join(input_file, "ROOT")
Tom Cherryd14b8952018-08-09 14:26:00 -0700345 d["root_fs_config"] = os.path.join(
Tao Bao410ad8b2018-08-24 12:08:38 -0700346 input_file, "META", "root_filesystem_config.txt")
Tao Bao84e75682015-07-19 02:38:53 -0700347
Tao Baof54216f2016-03-29 15:12:37 -0700348 # Redirect {system,vendor}_base_fs_file.
349 if "system_base_fs_file" in d:
350 basename = os.path.basename(d["system_base_fs_file"])
Tao Bao410ad8b2018-08-24 12:08:38 -0700351 system_base_fs_file = os.path.join(input_file, "META", basename)
Tao Baob079b502016-05-03 08:01:19 -0700352 if os.path.exists(system_base_fs_file):
353 d["system_base_fs_file"] = system_base_fs_file
354 else:
Tao Bao32fcdab2018-10-12 10:30:39 -0700355 logger.warning(
356 "Failed to find system base fs file: %s", system_base_fs_file)
Tao Baob079b502016-05-03 08:01:19 -0700357 del d["system_base_fs_file"]
Tao Baof54216f2016-03-29 15:12:37 -0700358
359 if "vendor_base_fs_file" in d:
360 basename = os.path.basename(d["vendor_base_fs_file"])
Tao Bao410ad8b2018-08-24 12:08:38 -0700361 vendor_base_fs_file = os.path.join(input_file, "META", basename)
Tao Baob079b502016-05-03 08:01:19 -0700362 if os.path.exists(vendor_base_fs_file):
363 d["vendor_base_fs_file"] = vendor_base_fs_file
364 else:
Tao Bao32fcdab2018-10-12 10:30:39 -0700365 logger.warning(
366 "Failed to find vendor base fs file: %s", vendor_base_fs_file)
Tao Baob079b502016-05-03 08:01:19 -0700367 del d["vendor_base_fs_file"]
Tao Baof54216f2016-03-29 15:12:37 -0700368
Doug Zongker37974732010-09-16 17:44:38 -0700369 def makeint(key):
370 if key in d:
371 d[key] = int(d[key], 0)
372
373 makeint("recovery_api_version")
374 makeint("blocksize")
375 makeint("system_size")
Daniel Rosenbergf4eabc32014-07-10 15:42:38 -0700376 makeint("vendor_size")
Doug Zongker37974732010-09-16 17:44:38 -0700377 makeint("userdata_size")
Ying Wang9f8e8db2011-11-04 11:37:01 -0700378 makeint("cache_size")
Doug Zongker37974732010-09-16 17:44:38 -0700379 makeint("recovery_size")
380 makeint("boot_size")
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800381 makeint("fstab_version")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700382
Tao Baoa57ab9f2018-08-24 12:08:38 -0700383 # We changed recovery.fstab path in Q, from ../RAMDISK/etc/recovery.fstab to
384 # ../RAMDISK/system/etc/recovery.fstab. LoadInfoDict() has to handle both
385 # cases, since it may load the info_dict from an old build (e.g. when
386 # generating incremental OTAs from that build).
Tao Bao76def242017-11-21 09:25:31 -0800387 system_root_image = d.get("system_root_image") == "true"
388 if d.get("no_recovery") != "true":
Tao Bao696bb332018-08-17 16:27:01 -0700389 recovery_fstab_path = "RECOVERY/RAMDISK/system/etc/recovery.fstab"
Tao Baob4adc062018-08-22 18:27:14 -0700390 if isinstance(input_file, zipfile.ZipFile):
391 if recovery_fstab_path not in input_file.namelist():
392 recovery_fstab_path = "RECOVERY/RAMDISK/etc/recovery.fstab"
393 else:
394 path = os.path.join(input_file, *recovery_fstab_path.split("/"))
395 if not os.path.exists(path):
396 recovery_fstab_path = "RECOVERY/RAMDISK/etc/recovery.fstab"
Tao Bao76def242017-11-21 09:25:31 -0800397 d["fstab"] = LoadRecoveryFSTab(
398 read_helper, d["fstab_version"], recovery_fstab_path, system_root_image)
Tao Baob4adc062018-08-22 18:27:14 -0700399
Tao Bao76def242017-11-21 09:25:31 -0800400 elif d.get("recovery_as_boot") == "true":
Tao Bao696bb332018-08-17 16:27:01 -0700401 recovery_fstab_path = "BOOT/RAMDISK/system/etc/recovery.fstab"
Tao Baob4adc062018-08-22 18:27:14 -0700402 if isinstance(input_file, zipfile.ZipFile):
403 if recovery_fstab_path not in input_file.namelist():
404 recovery_fstab_path = "BOOT/RAMDISK/etc/recovery.fstab"
405 else:
406 path = os.path.join(input_file, *recovery_fstab_path.split("/"))
407 if not os.path.exists(path):
408 recovery_fstab_path = "BOOT/RAMDISK/etc/recovery.fstab"
Tao Bao76def242017-11-21 09:25:31 -0800409 d["fstab"] = LoadRecoveryFSTab(
410 read_helper, d["fstab_version"], recovery_fstab_path, system_root_image)
Tao Baob4adc062018-08-22 18:27:14 -0700411
Tianjie Xucfa86222016-03-07 16:31:19 -0800412 else:
413 d["fstab"] = None
414
Tianjie Xu861f4132018-09-12 11:49:33 -0700415 # Tries to load the build props for all partitions with care_map, including
416 # system and vendor.
417 for partition in PARTITIONS_WITH_CARE_MAP:
418 d["{}.build.prop".format(partition)] = LoadBuildProp(
419 read_helper, "{}/build.prop".format(partition.upper()))
420 d["build.prop"] = d["system.build.prop"]
Tao Bao12d87fc2018-01-31 12:18:52 -0800421
422 # Set up the salt (based on fingerprint or thumbprint) that will be used when
423 # adding AVB footer.
424 if d.get("avb_enable") == "true":
425 fp = None
426 if "build.prop" in d:
427 build_prop = d["build.prop"]
428 if "ro.build.fingerprint" in build_prop:
429 fp = build_prop["ro.build.fingerprint"]
430 elif "ro.build.thumbprint" in build_prop:
431 fp = build_prop["ro.build.thumbprint"]
432 if fp:
433 d["avb_salt"] = sha256(fp).hexdigest()
434
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700435 return d
436
Tao Baod1de6f32017-03-01 16:38:48 -0800437
Tao Baobcd1d162017-08-26 13:10:26 -0700438def LoadBuildProp(read_helper, prop_file):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700439 try:
Tao Baobcd1d162017-08-26 13:10:26 -0700440 data = read_helper(prop_file)
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700441 except KeyError:
Tao Bao32fcdab2018-10-12 10:30:39 -0700442 logger.warning("Failed to read %s", prop_file)
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700443 data = ""
Michael Runge6e836112014-04-15 17:40:21 -0700444 return LoadDictionaryFromLines(data.split("\n"))
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700445
Tao Baod1de6f32017-03-01 16:38:48 -0800446
Michael Runge6e836112014-04-15 17:40:21 -0700447def LoadDictionaryFromLines(lines):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700448 d = {}
Michael Runge6e836112014-04-15 17:40:21 -0700449 for line in lines:
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700450 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700451 if not line or line.startswith("#"):
452 continue
Ying Wang114b46f2014-04-15 11:24:00 -0700453 if "=" in line:
454 name, value = line.split("=", 1)
455 d[name] = value
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700456 return d
457
Tao Baod1de6f32017-03-01 16:38:48 -0800458
Tianjie Xucfa86222016-03-07 16:31:19 -0800459def LoadRecoveryFSTab(read_helper, fstab_version, recovery_fstab_path,
460 system_root_image=False):
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700461 class Partition(object):
Tao Baod1de6f32017-03-01 16:38:48 -0800462 def __init__(self, mount_point, fs_type, device, length, context):
Dan Albert8b72aef2015-03-23 19:13:21 -0700463 self.mount_point = mount_point
464 self.fs_type = fs_type
465 self.device = device
466 self.length = length
Tao Bao548eb762015-06-10 12:32:41 -0700467 self.context = context
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700468
469 try:
Tianjie Xucfa86222016-03-07 16:31:19 -0800470 data = read_helper(recovery_fstab_path)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700471 except KeyError:
Tao Bao32fcdab2018-10-12 10:30:39 -0700472 logger.warning("Failed to find %s", recovery_fstab_path)
Jeff Davidson033fbe22011-10-26 18:08:09 -0700473 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700474
Tao Baod1de6f32017-03-01 16:38:48 -0800475 assert fstab_version == 2
476
477 d = {}
478 for line in data.split("\n"):
479 line = line.strip()
480 if not line or line.startswith("#"):
481 continue
482
483 # <src> <mnt_point> <type> <mnt_flags and options> <fs_mgr_flags>
484 pieces = line.split()
485 if len(pieces) != 5:
486 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
487
488 # Ignore entries that are managed by vold.
489 options = pieces[4]
490 if "voldmanaged=" in options:
491 continue
492
493 # It's a good line, parse it.
494 length = 0
495 options = options.split(",")
496 for i in options:
497 if i.startswith("length="):
498 length = int(i[7:])
Doug Zongker086cbb02011-02-17 15:54:20 -0800499 else:
Tao Baod1de6f32017-03-01 16:38:48 -0800500 # Ignore all unknown options in the unified fstab.
Dan Albert8b72aef2015-03-23 19:13:21 -0700501 continue
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800502
Tao Baod1de6f32017-03-01 16:38:48 -0800503 mount_flags = pieces[3]
504 # Honor the SELinux context if present.
505 context = None
506 for i in mount_flags.split(","):
507 if i.startswith("context="):
508 context = i
Doug Zongker086cbb02011-02-17 15:54:20 -0800509
Tao Baod1de6f32017-03-01 16:38:48 -0800510 mount_point = pieces[1]
511 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2],
512 device=pieces[0], length=length, context=context)
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800513
Daniel Rosenberge6853b02015-06-05 17:59:27 -0700514 # / is used for the system mount point when the root directory is included in
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700515 # system. Other areas assume system is always at "/system" so point /system
516 # at /.
Daniel Rosenberge6853b02015-06-05 17:59:27 -0700517 if system_root_image:
518 assert not d.has_key("/system") and d.has_key("/")
519 d["/system"] = d["/"]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700520 return d
521
522
Doug Zongker37974732010-09-16 17:44:38 -0700523def DumpInfoDict(d):
524 for k, v in sorted(d.items()):
Tao Bao32fcdab2018-10-12 10:30:39 -0700525 logger.info("%-25s = (%s) %s", k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700526
Dan Albert8b72aef2015-03-23 19:13:21 -0700527
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800528def AppendAVBSigningArgs(cmd, partition):
529 """Append signing arguments for avbtool."""
530 # e.g., "--key path/to/signing_key --algorithm SHA256_RSA4096"
531 key_path = OPTIONS.info_dict.get("avb_" + partition + "_key_path")
532 algorithm = OPTIONS.info_dict.get("avb_" + partition + "_algorithm")
533 if key_path and algorithm:
534 cmd.extend(["--key", key_path, "--algorithm", algorithm])
Tao Bao2b6dfd62017-09-27 17:17:43 -0700535 avb_salt = OPTIONS.info_dict.get("avb_salt")
536 # make_vbmeta_image doesn't like "--salt" (and it's not needed).
Tao Bao744c4c72018-08-20 21:09:07 -0700537 if avb_salt and not partition.startswith("vbmeta"):
Tao Bao2b6dfd62017-09-27 17:17:43 -0700538 cmd.extend(["--salt", avb_salt])
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800539
540
Tao Bao02a08592018-07-22 12:40:45 -0700541def GetAvbChainedPartitionArg(partition, info_dict, key=None):
542 """Constructs and returns the arg to build or verify a chained partition.
543
544 Args:
545 partition: The partition name.
546 info_dict: The info dict to look up the key info and rollback index
547 location.
548 key: The key to be used for building or verifying the partition. Defaults to
549 the key listed in info_dict.
550
551 Returns:
552 A string of form "partition:rollback_index_location:key" that can be used to
553 build or verify vbmeta image.
Tao Bao02a08592018-07-22 12:40:45 -0700554 """
555 if key is None:
556 key = info_dict["avb_" + partition + "_key_path"]
Tao Bao2cc0ca12019-03-15 10:44:43 -0700557 pubkey_path = ExtractAvbPublicKey(key)
Tao Bao02a08592018-07-22 12:40:45 -0700558 rollback_index_location = info_dict[
559 "avb_" + partition + "_rollback_index_location"]
560 return "{}:{}:{}".format(partition, rollback_index_location, pubkey_path)
561
562
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700563def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
Tao Baod42e97e2016-11-30 12:11:57 -0800564 has_ramdisk=False, two_step_image=False):
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700565 """Build a bootable image from the specified sourcedir.
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700566
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700567 Take a kernel, cmdline, and optionally a ramdisk directory from the input (in
Tao Baod42e97e2016-11-30 12:11:57 -0800568 'sourcedir'), and turn them into a boot image. 'two_step_image' indicates if
569 we are building a two-step special image (i.e. building a recovery image to
570 be loaded into /boot in two-step OTAs).
571
572 Return the image data, or None if sourcedir does not appear to contains files
573 for building the requested image.
574 """
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700575
576 def make_ramdisk():
577 ramdisk_img = tempfile.NamedTemporaryFile()
578
579 if os.access(fs_config_file, os.F_OK):
580 cmd = ["mkbootfs", "-f", fs_config_file,
581 os.path.join(sourcedir, "RAMDISK")]
582 else:
583 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")]
584 p1 = Run(cmd, stdout=subprocess.PIPE)
585 p2 = Run(["minigzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
586
587 p2.wait()
588 p1.wait()
589 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,)
590 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,)
591
592 return ramdisk_img
593
594 if not os.access(os.path.join(sourcedir, "kernel"), os.F_OK):
595 return None
596
597 if has_ramdisk and not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK):
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700598 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700599
Doug Zongkerd5131602012-08-02 14:46:42 -0700600 if info_dict is None:
601 info_dict = OPTIONS.info_dict
602
Doug Zongkereef39442009-04-02 12:14:19 -0700603 img = tempfile.NamedTemporaryFile()
604
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700605 if has_ramdisk:
606 ramdisk_img = make_ramdisk()
Doug Zongkereef39442009-04-02 12:14:19 -0700607
Bjorn Andersson612e2cd2012-11-25 16:53:44 -0800608 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set
609 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg"
610
611 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")]
Doug Zongker38a649f2009-06-17 09:07:09 -0700612
Benoit Fradina45a8682014-07-14 21:00:43 +0200613 fn = os.path.join(sourcedir, "second")
614 if os.access(fn, os.F_OK):
615 cmd.append("--second")
616 cmd.append(fn)
617
Hridya Valsaraju9683b2f2019-01-22 18:08:59 -0800618 fn = os.path.join(sourcedir, "dtb")
619 if os.access(fn, os.F_OK):
620 cmd.append("--dtb")
621 cmd.append(fn)
622
Doug Zongker171f1cd2009-06-15 22:36:37 -0700623 fn = os.path.join(sourcedir, "cmdline")
624 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700625 cmd.append("--cmdline")
626 cmd.append(open(fn).read().rstrip("\n"))
627
628 fn = os.path.join(sourcedir, "base")
629 if os.access(fn, os.F_OK):
630 cmd.append("--base")
631 cmd.append(open(fn).read().rstrip("\n"))
632
Ying Wang4de6b5b2010-08-25 14:29:34 -0700633 fn = os.path.join(sourcedir, "pagesize")
634 if os.access(fn, os.F_OK):
635 cmd.append("--pagesize")
636 cmd.append(open(fn).read().rstrip("\n"))
637
Tao Bao76def242017-11-21 09:25:31 -0800638 args = info_dict.get("mkbootimg_args")
Doug Zongkerd5131602012-08-02 14:46:42 -0700639 if args and args.strip():
Jianxun Zhang09849492013-04-17 15:19:19 -0700640 cmd.extend(shlex.split(args))
Doug Zongkerd5131602012-08-02 14:46:42 -0700641
Tao Bao76def242017-11-21 09:25:31 -0800642 args = info_dict.get("mkbootimg_version_args")
Sami Tolvanen3303d902016-03-15 16:49:30 +0000643 if args and args.strip():
644 cmd.extend(shlex.split(args))
645
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700646 if has_ramdisk:
647 cmd.extend(["--ramdisk", ramdisk_img.name])
648
Tao Baod95e9fd2015-03-29 23:07:41 -0700649 img_unsigned = None
Tao Bao76def242017-11-21 09:25:31 -0800650 if info_dict.get("vboot"):
Tao Baod95e9fd2015-03-29 23:07:41 -0700651 img_unsigned = tempfile.NamedTemporaryFile()
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700652 cmd.extend(["--output", img_unsigned.name])
Tao Baod95e9fd2015-03-29 23:07:41 -0700653 else:
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700654 cmd.extend(["--output", img.name])
Doug Zongker38a649f2009-06-17 09:07:09 -0700655
Tao Baobf70c3182017-07-11 17:27:55 -0700656 # "boot" or "recovery", without extension.
657 partition_name = os.path.basename(sourcedir).lower()
658
Chen, ZhiminX752439b2018-09-23 22:10:47 +0800659 if partition_name == "recovery":
660 if info_dict.get("include_recovery_dtbo") == "true":
661 fn = os.path.join(sourcedir, "recovery_dtbo")
662 cmd.extend(["--recovery_dtbo", fn])
663 if info_dict.get("include_recovery_acpio") == "true":
664 fn = os.path.join(sourcedir, "recovery_acpio")
665 cmd.extend(["--recovery_acpio", fn])
Hridya Valsarajue74a38b2018-03-21 12:15:11 -0700666
Tao Bao986ee862018-10-04 15:46:16 -0700667 RunAndCheckOutput(cmd)
Doug Zongkereef39442009-04-02 12:14:19 -0700668
Tao Bao76def242017-11-21 09:25:31 -0800669 if (info_dict.get("boot_signer") == "true" and
670 info_dict.get("verity_key")):
Tao Baod42e97e2016-11-30 12:11:57 -0800671 # Hard-code the path as "/boot" for two-step special recovery image (which
672 # will be loaded into /boot during the two-step OTA).
673 if two_step_image:
674 path = "/boot"
675 else:
Tao Baobf70c3182017-07-11 17:27:55 -0700676 path = "/" + partition_name
Baligh Uddin601ddea2015-06-09 15:48:14 -0700677 cmd = [OPTIONS.boot_signer_path]
678 cmd.extend(OPTIONS.boot_signer_args)
679 cmd.extend([path, img.name,
680 info_dict["verity_key"] + ".pk8",
681 info_dict["verity_key"] + ".x509.pem", img.name])
Tao Bao986ee862018-10-04 15:46:16 -0700682 RunAndCheckOutput(cmd)
Geremy Condra95ebe7a2014-08-19 17:27:56 -0700683
Tao Baod95e9fd2015-03-29 23:07:41 -0700684 # Sign the image if vboot is non-empty.
Tao Bao76def242017-11-21 09:25:31 -0800685 elif info_dict.get("vboot"):
Tao Baobf70c3182017-07-11 17:27:55 -0700686 path = "/" + partition_name
Tao Baod95e9fd2015-03-29 23:07:41 -0700687 img_keyblock = tempfile.NamedTemporaryFile()
Tao Bao4f104d12017-02-17 23:21:31 -0800688 # We have switched from the prebuilt futility binary to using the tool
689 # (futility-host) built from the source. Override the setting in the old
690 # TF.zip.
691 futility = info_dict["futility"]
692 if futility.startswith("prebuilts/"):
693 futility = "futility-host"
694 cmd = [info_dict["vboot_signer_cmd"], futility,
Tao Baod95e9fd2015-03-29 23:07:41 -0700695 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk",
Furquan Shaikh852b8de2015-08-10 11:43:45 -0700696 info_dict["vboot_key"] + ".vbprivk",
697 info_dict["vboot_subkey"] + ".vbprivk",
698 img_keyblock.name,
Tao Baod95e9fd2015-03-29 23:07:41 -0700699 img.name]
Tao Bao986ee862018-10-04 15:46:16 -0700700 RunAndCheckOutput(cmd)
Tao Baod95e9fd2015-03-29 23:07:41 -0700701
Tao Baof3282b42015-04-01 11:21:55 -0700702 # Clean up the temp files.
703 img_unsigned.close()
704 img_keyblock.close()
705
David Zeuthen8fecb282017-12-01 16:24:01 -0500706 # AVB: if enabled, calculate and add hash to boot.img or recovery.img.
Bowgo Tsai3e599ea2017-05-26 18:30:04 +0800707 if info_dict.get("avb_enable") == "true":
Tao Baof88e0ce2019-03-18 14:01:38 -0700708 avbtool = info_dict["avb_avbtool"]
David Zeuthen8fecb282017-12-01 16:24:01 -0500709 part_size = info_dict[partition_name + "_size"]
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400710 cmd = [avbtool, "add_hash_footer", "--image", img.name,
Tao Baobf70c3182017-07-11 17:27:55 -0700711 "--partition_size", str(part_size), "--partition_name",
712 partition_name]
713 AppendAVBSigningArgs(cmd, partition_name)
David Zeuthen8fecb282017-12-01 16:24:01 -0500714 args = info_dict.get("avb_" + partition_name + "_add_hash_footer_args")
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400715 if args and args.strip():
716 cmd.extend(shlex.split(args))
Tao Bao986ee862018-10-04 15:46:16 -0700717 RunAndCheckOutput(cmd)
David Zeuthend995f4b2016-01-29 16:59:17 -0500718
719 img.seek(os.SEEK_SET, 0)
720 data = img.read()
721
722 if has_ramdisk:
723 ramdisk_img.close()
724 img.close()
725
726 return data
727
728
Doug Zongkerd5131602012-08-02 14:46:42 -0700729def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,
Tao Baod42e97e2016-11-30 12:11:57 -0800730 info_dict=None, two_step_image=False):
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700731 """Return a File object with the desired bootable image.
732
733 Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 'prebuilt_name',
734 otherwise look for it under 'unpack_dir'/IMAGES, otherwise construct it from
735 the source files in 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700736
Doug Zongker55d93282011-01-25 17:03:34 -0800737 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
738 if os.path.exists(prebuilt_path):
Tao Bao32fcdab2018-10-12 10:30:39 -0700739 logger.info("using prebuilt %s from BOOTABLE_IMAGES...", prebuilt_name)
Doug Zongker55d93282011-01-25 17:03:34 -0800740 return File.FromLocalFile(name, prebuilt_path)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700741
742 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name)
743 if os.path.exists(prebuilt_path):
Tao Bao32fcdab2018-10-12 10:30:39 -0700744 logger.info("using prebuilt %s from IMAGES...", prebuilt_name)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700745 return File.FromLocalFile(name, prebuilt_path)
746
Tao Bao32fcdab2018-10-12 10:30:39 -0700747 logger.info("building image from target_files %s...", tree_subdir)
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700748
749 if info_dict is None:
750 info_dict = OPTIONS.info_dict
751
752 # With system_root_image == "true", we don't pack ramdisk into the boot image.
Daniel Rosenberg40ef35b2015-11-10 19:21:34 -0800753 # Unless "recovery_as_boot" is specified, in which case we carry the ramdisk
754 # for recovery.
755 has_ramdisk = (info_dict.get("system_root_image") != "true" or
756 prebuilt_name != "boot.img" or
757 info_dict.get("recovery_as_boot") == "true")
Tao Bao7a5bf8a2015-07-21 18:01:20 -0700758
Doug Zongker6f1d0312014-08-22 08:07:12 -0700759 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt"
David Zeuthen2ce63ed2016-09-15 13:43:54 -0400760 data = _BuildBootableImage(os.path.join(unpack_dir, tree_subdir),
761 os.path.join(unpack_dir, fs_config),
Tao Baod42e97e2016-11-30 12:11:57 -0800762 info_dict, has_ramdisk, two_step_image)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700763 if data:
764 return File(name, data)
765 return None
Doug Zongker55d93282011-01-25 17:03:34 -0800766
Doug Zongkereef39442009-04-02 12:14:19 -0700767
Narayan Kamatha07bf042017-08-14 14:49:21 +0100768def Gunzip(in_filename, out_filename):
Tao Bao76def242017-11-21 09:25:31 -0800769 """Gunzips the given gzip compressed file to a given output file."""
770 with gzip.open(in_filename, "rb") as in_file, \
771 open(out_filename, "wb") as out_file:
Narayan Kamatha07bf042017-08-14 14:49:21 +0100772 shutil.copyfileobj(in_file, out_file)
773
774
Tao Bao0ff15de2019-03-20 11:26:06 -0700775def UnzipToDir(filename, dirname, patterns=None):
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800776 """Unzips the archive to the given directory.
777
778 Args:
779 filename: The name of the zip file to unzip.
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800780 dirname: Where the unziped files will land.
Tao Bao0ff15de2019-03-20 11:26:06 -0700781 patterns: Files to unzip from the archive. If omitted, will unzip the entire
782 archvie. Non-matching patterns will be filtered out. If there's no match
783 after the filtering, no file will be unzipped.
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800784 """
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800785 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
Tao Bao0ff15de2019-03-20 11:26:06 -0700786 if patterns is not None:
787 # Filter out non-matching patterns. unzip will complain otherwise.
788 with zipfile.ZipFile(filename) as input_zip:
789 names = input_zip.namelist()
790 filtered = [
791 pattern for pattern in patterns if fnmatch.filter(names, pattern)]
792
793 # There isn't any matching files. Don't unzip anything.
794 if not filtered:
795 return
796 cmd.extend(filtered)
797
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800798 RunAndCheckOutput(cmd)
799
800
Doug Zongker75f17362009-12-08 13:46:44 -0800801def UnzipTemp(filename, pattern=None):
Tao Bao1c830bf2017-12-25 10:43:47 -0800802 """Unzips the given archive into a temporary directory and returns the name.
Doug Zongker55d93282011-01-25 17:03:34 -0800803
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800804 Args:
805 filename: If filename is of the form "foo.zip+bar.zip", unzip foo.zip into
806 a temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
807
808 pattern: Files to unzip from the archive. If omitted, will unzip the entire
809 archvie.
Doug Zongker55d93282011-01-25 17:03:34 -0800810
Tao Bao1c830bf2017-12-25 10:43:47 -0800811 Returns:
Tao Baodba59ee2018-01-09 13:21:02 -0800812 The name of the temporary directory.
Doug Zongker55d93282011-01-25 17:03:34 -0800813 """
Doug Zongkereef39442009-04-02 12:14:19 -0700814
Tao Bao1c830bf2017-12-25 10:43:47 -0800815 tmp = MakeTempDir(prefix="targetfiles-")
Doug Zongker55d93282011-01-25 17:03:34 -0800816 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
817 if m:
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800818 UnzipToDir(m.group(1), tmp, pattern)
819 UnzipToDir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"), pattern)
Doug Zongker55d93282011-01-25 17:03:34 -0800820 filename = m.group(1)
821 else:
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800822 UnzipToDir(filename, tmp, pattern)
Doug Zongker55d93282011-01-25 17:03:34 -0800823
Tao Baodba59ee2018-01-09 13:21:02 -0800824 return tmp
Doug Zongkereef39442009-04-02 12:14:19 -0700825
826
Tianjie Xu67c7cbb2018-08-30 00:32:07 -0700827def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks,
828 hashtree_info_generator=None):
Tao Baoc765cca2018-01-31 17:32:40 -0800829 """Returns a SparseImage object suitable for passing to BlockImageDiff.
830
831 This function loads the specified sparse image from the given path, and
832 performs additional processing for OTA purpose. For example, it always adds
833 block 0 to clobbered blocks list. It also detects files that cannot be
834 reconstructed from the block list, for whom we should avoid applying imgdiff.
835
836 Args:
837 which: The partition name, which must be "system" or "vendor".
838 tmpdir: The directory that contains the prebuilt image and block map file.
839 input_zip: The target-files ZIP archive.
Tao Baoe709b092018-02-07 12:40:00 -0800840 allow_shared_blocks: Whether having shared blocks is allowed.
Tianjie Xu67c7cbb2018-08-30 00:32:07 -0700841 hashtree_info_generator: If present, generates the hashtree_info for this
842 sparse image.
Tao Baoc765cca2018-01-31 17:32:40 -0800843 Returns:
844 A SparseImage object, with file_map info loaded.
845 """
846 assert which in ("system", "vendor")
847
848 path = os.path.join(tmpdir, "IMAGES", which + ".img")
849 mappath = os.path.join(tmpdir, "IMAGES", which + ".map")
850
851 # The image and map files must have been created prior to calling
852 # ota_from_target_files.py (since LMP).
853 assert os.path.exists(path) and os.path.exists(mappath)
854
855 # In ext4 filesystems, block 0 might be changed even being mounted R/O. We add
856 # it to clobbered_blocks so that it will be written to the target
857 # unconditionally. Note that they are still part of care_map. (Bug: 20939131)
858 clobbered_blocks = "0"
859
Tianjie Xu67c7cbb2018-08-30 00:32:07 -0700860 image = sparse_img.SparseImage(
861 path, mappath, clobbered_blocks, allow_shared_blocks=allow_shared_blocks,
862 hashtree_info_generator=hashtree_info_generator)
Tao Baoc765cca2018-01-31 17:32:40 -0800863
864 # block.map may contain less blocks, because mke2fs may skip allocating blocks
865 # if they contain all zeros. We can't reconstruct such a file from its block
866 # list. Tag such entries accordingly. (Bug: 65213616)
867 for entry in image.file_map:
Tao Baoc765cca2018-01-31 17:32:40 -0800868 # Skip artificial names, such as "__ZERO", "__NONZERO-1".
Tao Baod3554e62018-07-10 15:31:22 -0700869 if not entry.startswith('/'):
Tao Baoc765cca2018-01-31 17:32:40 -0800870 continue
871
Tom Cherryd14b8952018-08-09 14:26:00 -0700872 # "/system/framework/am.jar" => "SYSTEM/framework/am.jar". Note that the
873 # filename listed in system.map may contain an additional leading slash
874 # (i.e. "//system/framework/am.jar"). Using lstrip to get consistent
875 # results.
Tao Baod3554e62018-07-10 15:31:22 -0700876 arcname = string.replace(entry, which, which.upper(), 1).lstrip('/')
877
Tom Cherryd14b8952018-08-09 14:26:00 -0700878 # Special handling another case, where files not under /system
879 # (e.g. "/sbin/charger") are packed under ROOT/ in a target_files.zip.
Tao Baod3554e62018-07-10 15:31:22 -0700880 if which == 'system' and not arcname.startswith('SYSTEM'):
881 arcname = 'ROOT/' + arcname
882
883 assert arcname in input_zip.namelist(), \
884 "Failed to find the ZIP entry for {}".format(entry)
885
Tao Baoc765cca2018-01-31 17:32:40 -0800886 info = input_zip.getinfo(arcname)
887 ranges = image.file_map[entry]
Tao Baoe709b092018-02-07 12:40:00 -0800888
889 # If a RangeSet has been tagged as using shared blocks while loading the
Tao Bao2a20f342018-12-03 15:08:23 -0800890 # image, check the original block list to determine its completeness. Note
891 # that the 'incomplete' flag would be tagged to the original RangeSet only.
Tao Baoe709b092018-02-07 12:40:00 -0800892 if ranges.extra.get('uses_shared_blocks'):
Tao Bao2a20f342018-12-03 15:08:23 -0800893 ranges = ranges.extra['uses_shared_blocks']
Tao Baoe709b092018-02-07 12:40:00 -0800894
Tao Baoc765cca2018-01-31 17:32:40 -0800895 if RoundUpTo4K(info.file_size) > ranges.size() * 4096:
896 ranges.extra['incomplete'] = True
897
898 return image
899
900
Doug Zongkereef39442009-04-02 12:14:19 -0700901def GetKeyPasswords(keylist):
902 """Given a list of keys, prompt the user to enter passwords for
903 those which require them. Return a {key: password} dict. password
904 will be None if the key has no password."""
905
Doug Zongker8ce7c252009-05-22 13:34:54 -0700906 no_passwords = []
907 need_passwords = []
T.R. Fullhart37e10522013-03-18 10:31:26 -0700908 key_passwords = {}
Doug Zongkereef39442009-04-02 12:14:19 -0700909 devnull = open("/dev/null", "w+b")
910 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800911 # We don't need a password for things that aren't really keys.
912 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700913 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700914 continue
915
T.R. Fullhart37e10522013-03-18 10:31:26 -0700916 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
Doug Zongker602a84e2009-06-18 08:35:12 -0700917 "-inform", "DER", "-nocrypt"],
918 stdin=devnull.fileno(),
919 stdout=devnull.fileno(),
920 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700921 p.communicate()
922 if p.returncode == 0:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700923 # Definitely an unencrypted key.
Doug Zongker8ce7c252009-05-22 13:34:54 -0700924 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700925 else:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700926 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
927 "-inform", "DER", "-passin", "pass:"],
928 stdin=devnull.fileno(),
929 stdout=devnull.fileno(),
930 stderr=subprocess.PIPE)
Dan Albert8b72aef2015-03-23 19:13:21 -0700931 _, stderr = p.communicate()
T.R. Fullhart37e10522013-03-18 10:31:26 -0700932 if p.returncode == 0:
933 # Encrypted key with empty string as password.
934 key_passwords[k] = ''
935 elif stderr.startswith('Error decrypting key'):
936 # Definitely encrypted key.
937 # It would have said "Error reading key" if it didn't parse correctly.
938 need_passwords.append(k)
939 else:
940 # Potentially, a type of key that openssl doesn't understand.
941 # We'll let the routines in signapk.jar handle it.
942 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700943 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700944
T.R. Fullhart37e10522013-03-18 10:31:26 -0700945 key_passwords.update(PasswordManager().GetPasswords(need_passwords))
Tao Bao76def242017-11-21 09:25:31 -0800946 key_passwords.update(dict.fromkeys(no_passwords))
Doug Zongkereef39442009-04-02 12:14:19 -0700947 return key_passwords
948
949
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800950def GetMinSdkVersion(apk_name):
Tao Baof47bf0f2018-03-21 23:28:51 -0700951 """Gets the minSdkVersion declared in the APK.
952
953 It calls 'aapt' to query the embedded minSdkVersion from the given APK file.
954 This can be both a decimal number (API Level) or a codename.
955
956 Args:
957 apk_name: The APK filename.
958
959 Returns:
960 The parsed SDK version string.
961
962 Raises:
963 ExternalError: On failing to obtain the min SDK version.
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800964 """
Tao Baof47bf0f2018-03-21 23:28:51 -0700965 proc = Run(
966 ["aapt", "dump", "badging", apk_name], stdout=subprocess.PIPE,
967 stderr=subprocess.PIPE)
968 stdoutdata, stderrdata = proc.communicate()
969 if proc.returncode != 0:
970 raise ExternalError(
971 "Failed to obtain minSdkVersion: aapt return code {}:\n{}\n{}".format(
972 proc.returncode, stdoutdata, stderrdata))
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800973
Tao Baof47bf0f2018-03-21 23:28:51 -0700974 for line in stdoutdata.split("\n"):
975 # Looking for lines such as sdkVersion:'23' or sdkVersion:'M'.
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800976 m = re.match(r'sdkVersion:\'([^\']*)\'', line)
977 if m:
978 return m.group(1)
979 raise ExternalError("No minSdkVersion returned by aapt")
980
981
982def GetMinSdkVersionInt(apk_name, codename_to_api_level_map):
Tao Baof47bf0f2018-03-21 23:28:51 -0700983 """Returns the minSdkVersion declared in the APK as a number (API Level).
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800984
Tao Baof47bf0f2018-03-21 23:28:51 -0700985 If minSdkVersion is set to a codename, it is translated to a number using the
986 provided map.
987
988 Args:
989 apk_name: The APK filename.
990
991 Returns:
992 The parsed SDK version number.
993
994 Raises:
995 ExternalError: On failing to get the min SDK version number.
996 """
Alex Klyubin2cfd1d12016-01-13 10:32:47 -0800997 version = GetMinSdkVersion(apk_name)
998 try:
999 return int(version)
1000 except ValueError:
1001 # Not a decimal number. Codename?
1002 if version in codename_to_api_level_map:
1003 return codename_to_api_level_map[version]
1004 else:
Tao Baof47bf0f2018-03-21 23:28:51 -07001005 raise ExternalError(
1006 "Unknown minSdkVersion: '{}'. Known codenames: {}".format(
1007 version, codename_to_api_level_map))
Alex Klyubin2cfd1d12016-01-13 10:32:47 -08001008
1009
1010def SignFile(input_name, output_name, key, password, min_api_level=None,
Tao Bao76def242017-11-21 09:25:31 -08001011 codename_to_api_level_map=None, whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -07001012 """Sign the input_name zip/jar/apk, producing output_name. Use the
1013 given key and password (the latter may be None if the key does not
1014 have a password.
1015
Doug Zongker951495f2009-08-14 12:44:19 -07001016 If whole_file is true, use the "-w" option to SignApk to embed a
1017 signature that covers the whole file in the archive comment of the
1018 zip file.
Alex Klyubin2cfd1d12016-01-13 10:32:47 -08001019
1020 min_api_level is the API Level (int) of the oldest platform this file may end
1021 up on. If not specified for an APK, the API Level is obtained by interpreting
1022 the minSdkVersion attribute of the APK's AndroidManifest.xml.
1023
1024 codename_to_api_level_map is needed to translate the codename which may be
1025 encountered as the APK's minSdkVersion.
Doug Zongkereef39442009-04-02 12:14:19 -07001026 """
Tao Bao76def242017-11-21 09:25:31 -08001027 if codename_to_api_level_map is None:
1028 codename_to_api_level_map = {}
Doug Zongker951495f2009-08-14 12:44:19 -07001029
Alex Klyubin9667b182015-12-10 13:38:50 -08001030 java_library_path = os.path.join(
1031 OPTIONS.search_path, OPTIONS.signapk_shared_library_path)
1032
Tao Baoe95540e2016-11-08 12:08:53 -08001033 cmd = ([OPTIONS.java_path] + OPTIONS.java_args +
1034 ["-Djava.library.path=" + java_library_path,
1035 "-jar", os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] +
1036 OPTIONS.extra_signapk_args)
Doug Zongker951495f2009-08-14 12:44:19 -07001037 if whole_file:
1038 cmd.append("-w")
Alex Klyubin2cfd1d12016-01-13 10:32:47 -08001039
1040 min_sdk_version = min_api_level
1041 if min_sdk_version is None:
1042 if not whole_file:
1043 min_sdk_version = GetMinSdkVersionInt(
1044 input_name, codename_to_api_level_map)
1045 if min_sdk_version is not None:
1046 cmd.extend(["--min-sdk-version", str(min_sdk_version)])
1047
T.R. Fullhart37e10522013-03-18 10:31:26 -07001048 cmd.extend([key + OPTIONS.public_key_suffix,
1049 key + OPTIONS.private_key_suffix,
Alex Klyubineb756d72015-12-04 09:21:08 -08001050 input_name, output_name])
Doug Zongker951495f2009-08-14 12:44:19 -07001051
Tao Bao73dd4f42018-10-04 16:25:33 -07001052 proc = Run(cmd, stdin=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -07001053 if password is not None:
1054 password += "\n"
Tao Bao73dd4f42018-10-04 16:25:33 -07001055 stdoutdata, _ = proc.communicate(password)
1056 if proc.returncode != 0:
Tao Bao80921982018-03-21 21:02:19 -07001057 raise ExternalError(
1058 "Failed to run signapk.jar: return code {}:\n{}".format(
Tao Bao73dd4f42018-10-04 16:25:33 -07001059 proc.returncode, stdoutdata))
Doug Zongkereef39442009-04-02 12:14:19 -07001060
Doug Zongkereef39442009-04-02 12:14:19 -07001061
Doug Zongker37974732010-09-16 17:44:38 -07001062def CheckSize(data, target, info_dict):
Tao Bao9dd909e2017-11-14 11:27:32 -08001063 """Checks the data string passed against the max size limit.
Doug Zongkerc77a9ad2010-09-16 11:28:43 -07001064
Tao Bao9dd909e2017-11-14 11:27:32 -08001065 For non-AVB images, raise exception if the data is too big. Print a warning
1066 if the data is nearing the maximum size.
1067
1068 For AVB images, the actual image size should be identical to the limit.
1069
1070 Args:
1071 data: A string that contains all the data for the partition.
1072 target: The partition name. The ".img" suffix is optional.
1073 info_dict: The dict to be looked up for relevant info.
1074 """
Dan Albert8b72aef2015-03-23 19:13:21 -07001075 if target.endswith(".img"):
1076 target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -07001077 mount_point = "/" + target
1078
Ying Wangf8824af2014-06-03 14:07:27 -07001079 fs_type = None
1080 limit = None
Doug Zongker9ce0fb62010-09-20 18:04:41 -07001081 if info_dict["fstab"]:
Dan Albert8b72aef2015-03-23 19:13:21 -07001082 if mount_point == "/userdata":
1083 mount_point = "/data"
Doug Zongker9ce0fb62010-09-20 18:04:41 -07001084 p = info_dict["fstab"][mount_point]
1085 fs_type = p.fs_type
Andrew Boie0f9aec82012-02-14 09:32:52 -08001086 device = p.device
1087 if "/" in device:
1088 device = device[device.rfind("/")+1:]
Tao Bao76def242017-11-21 09:25:31 -08001089 limit = info_dict.get(device + "_size")
Dan Albert8b72aef2015-03-23 19:13:21 -07001090 if not fs_type or not limit:
1091 return
Doug Zongkereef39442009-04-02 12:14:19 -07001092
Andrew Boie0f9aec82012-02-14 09:32:52 -08001093 size = len(data)
Tao Bao9dd909e2017-11-14 11:27:32 -08001094 # target could be 'userdata' or 'cache'. They should follow the non-AVB image
1095 # path.
1096 if info_dict.get("avb_enable") == "true" and target in AVB_PARTITIONS:
1097 if size != limit:
1098 raise ExternalError(
1099 "Mismatching image size for %s: expected %d actual %d" % (
1100 target, limit, size))
1101 else:
1102 pct = float(size) * 100.0 / limit
1103 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
1104 if pct >= 99.0:
1105 raise ExternalError(msg)
1106 elif pct >= 95.0:
Tao Bao32fcdab2018-10-12 10:30:39 -07001107 logger.warning("\n WARNING: %s\n", msg)
1108 else:
1109 logger.info(" %s", msg)
Doug Zongkereef39442009-04-02 12:14:19 -07001110
1111
Doug Zongkerf6a53aa2009-12-15 15:06:55 -08001112def ReadApkCerts(tf_zip):
Tao Bao818ddf52018-01-05 11:17:34 -08001113 """Parses the APK certs info from a given target-files zip.
1114
1115 Given a target-files ZipFile, parses the META/apkcerts.txt entry and returns a
1116 tuple with the following elements: (1) a dictionary that maps packages to
1117 certs (based on the "certificate" and "private_key" attributes in the file;
1118 (2) a string representing the extension of compressed APKs in the target files
1119 (e.g ".gz", ".bro").
1120
1121 Args:
1122 tf_zip: The input target_files ZipFile (already open).
1123
1124 Returns:
1125 (certmap, ext): certmap is a dictionary that maps packages to certs; ext is
1126 the extension string of compressed APKs (e.g. ".gz"), or None if there's
1127 no compressed APKs.
1128 """
Doug Zongkerf6a53aa2009-12-15 15:06:55 -08001129 certmap = {}
Narayan Kamatha07bf042017-08-14 14:49:21 +01001130 compressed_extension = None
1131
Tao Bao0f990332017-09-08 19:02:54 -07001132 # META/apkcerts.txt contains the info for _all_ the packages known at build
1133 # time. Filter out the ones that are not installed.
1134 installed_files = set()
1135 for name in tf_zip.namelist():
1136 basename = os.path.basename(name)
1137 if basename:
1138 installed_files.add(basename)
1139
Doug Zongkerf6a53aa2009-12-15 15:06:55 -08001140 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
1141 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -07001142 if not line:
1143 continue
Tao Bao818ddf52018-01-05 11:17:34 -08001144 m = re.match(
1145 r'^name="(?P<NAME>.*)"\s+certificate="(?P<CERT>.*)"\s+'
1146 r'private_key="(?P<PRIVKEY>.*?)"(\s+compressed="(?P<COMPRESSED>.*)")?$',
1147 line)
1148 if not m:
1149 continue
Narayan Kamatha07bf042017-08-14 14:49:21 +01001150
Tao Bao818ddf52018-01-05 11:17:34 -08001151 matches = m.groupdict()
1152 cert = matches["CERT"]
1153 privkey = matches["PRIVKEY"]
1154 name = matches["NAME"]
1155 this_compressed_extension = matches["COMPRESSED"]
1156
1157 public_key_suffix_len = len(OPTIONS.public_key_suffix)
1158 private_key_suffix_len = len(OPTIONS.private_key_suffix)
1159 if cert in SPECIAL_CERT_STRINGS and not privkey:
1160 certmap[name] = cert
1161 elif (cert.endswith(OPTIONS.public_key_suffix) and
1162 privkey.endswith(OPTIONS.private_key_suffix) and
1163 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]):
1164 certmap[name] = cert[:-public_key_suffix_len]
1165 else:
1166 raise ValueError("Failed to parse line from apkcerts.txt:\n" + line)
1167
1168 if not this_compressed_extension:
1169 continue
1170
1171 # Only count the installed files.
1172 filename = name + '.' + this_compressed_extension
1173 if filename not in installed_files:
1174 continue
1175
1176 # Make sure that all the values in the compression map have the same
1177 # extension. We don't support multiple compression methods in the same
1178 # system image.
1179 if compressed_extension:
1180 if this_compressed_extension != compressed_extension:
1181 raise ValueError(
1182 "Multiple compressed extensions: {} vs {}".format(
1183 compressed_extension, this_compressed_extension))
1184 else:
1185 compressed_extension = this_compressed_extension
1186
1187 return (certmap,
1188 ("." + compressed_extension) if compressed_extension else None)
Doug Zongkerf6a53aa2009-12-15 15:06:55 -08001189
1190
Doug Zongkereef39442009-04-02 12:14:19 -07001191COMMON_DOCSTRING = """
Tao Bao30df8b42018-04-23 15:32:53 -07001192Global options
1193
1194 -p (--path) <dir>
1195 Prepend <dir>/bin to the list of places to search for binaries run by this
1196 script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -07001197
Doug Zongker05d3dea2009-06-22 11:32:31 -07001198 -s (--device_specific) <file>
Tao Bao30df8b42018-04-23 15:32:53 -07001199 Path to the Python module containing device-specific releasetools code.
Doug Zongker05d3dea2009-06-22 11:32:31 -07001200
Tao Bao30df8b42018-04-23 15:32:53 -07001201 -x (--extra) <key=value>
1202 Add a key/value pair to the 'extras' dict, which device-specific extension
1203 code may look at.
Doug Zongker8bec09e2009-11-30 15:37:14 -08001204
Doug Zongkereef39442009-04-02 12:14:19 -07001205 -v (--verbose)
1206 Show command lines being executed.
1207
1208 -h (--help)
1209 Display this usage message and exit.
1210"""
1211
1212def Usage(docstring):
Tao Bao89fbb0f2017-01-10 10:47:58 -08001213 print(docstring.rstrip("\n"))
1214 print(COMMON_DOCSTRING)
Doug Zongkereef39442009-04-02 12:14:19 -07001215
1216
1217def ParseOptions(argv,
1218 docstring,
1219 extra_opts="", extra_long_opts=(),
1220 extra_option_handler=None):
1221 """Parse the options in argv and return any arguments that aren't
1222 flags. docstring is the calling module's docstring, to be displayed
1223 for errors and -h. extra_opts and extra_long_opts are for flags
1224 defined by the caller, which are processed by passing them to
1225 extra_option_handler."""
1226
1227 try:
1228 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -08001229 argv, "hvp:s:x:" + extra_opts,
Alex Klyubin9667b182015-12-10 13:38:50 -08001230 ["help", "verbose", "path=", "signapk_path=",
1231 "signapk_shared_library_path=", "extra_signapk_args=",
Baligh Uddinbdc2e312014-09-05 17:36:20 -07001232 "java_path=", "java_args=", "public_key_suffix=",
Baligh Uddin601ddea2015-06-09 15:48:14 -07001233 "private_key_suffix=", "boot_signer_path=", "boot_signer_args=",
1234 "verity_signer_path=", "verity_signer_args=", "device_specific=",
Baligh Uddine2048682014-11-20 09:52:05 -08001235 "extra="] +
T.R. Fullhart37e10522013-03-18 10:31:26 -07001236 list(extra_long_opts))
Dan Albert8b72aef2015-03-23 19:13:21 -07001237 except getopt.GetoptError as err:
Doug Zongkereef39442009-04-02 12:14:19 -07001238 Usage(docstring)
Tao Bao89fbb0f2017-01-10 10:47:58 -08001239 print("**", str(err), "**")
Doug Zongkereef39442009-04-02 12:14:19 -07001240 sys.exit(2)
1241
Doug Zongkereef39442009-04-02 12:14:19 -07001242 for o, a in opts:
1243 if o in ("-h", "--help"):
1244 Usage(docstring)
1245 sys.exit()
1246 elif o in ("-v", "--verbose"):
1247 OPTIONS.verbose = True
1248 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -07001249 OPTIONS.search_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -07001250 elif o in ("--signapk_path",):
1251 OPTIONS.signapk_path = a
Alex Klyubin9667b182015-12-10 13:38:50 -08001252 elif o in ("--signapk_shared_library_path",):
1253 OPTIONS.signapk_shared_library_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -07001254 elif o in ("--extra_signapk_args",):
1255 OPTIONS.extra_signapk_args = shlex.split(a)
1256 elif o in ("--java_path",):
1257 OPTIONS.java_path = a
Baligh Uddin339ee492014-09-05 11:18:07 -07001258 elif o in ("--java_args",):
Tao Baoe95540e2016-11-08 12:08:53 -08001259 OPTIONS.java_args = shlex.split(a)
T.R. Fullhart37e10522013-03-18 10:31:26 -07001260 elif o in ("--public_key_suffix",):
1261 OPTIONS.public_key_suffix = a
1262 elif o in ("--private_key_suffix",):
1263 OPTIONS.private_key_suffix = a
Baligh Uddine2048682014-11-20 09:52:05 -08001264 elif o in ("--boot_signer_path",):
1265 OPTIONS.boot_signer_path = a
Baligh Uddin601ddea2015-06-09 15:48:14 -07001266 elif o in ("--boot_signer_args",):
1267 OPTIONS.boot_signer_args = shlex.split(a)
1268 elif o in ("--verity_signer_path",):
1269 OPTIONS.verity_signer_path = a
1270 elif o in ("--verity_signer_args",):
1271 OPTIONS.verity_signer_args = shlex.split(a)
Doug Zongker05d3dea2009-06-22 11:32:31 -07001272 elif o in ("-s", "--device_specific"):
1273 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -08001274 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -08001275 key, value = a.split("=", 1)
1276 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -07001277 else:
1278 if extra_option_handler is None or not extra_option_handler(o, a):
1279 assert False, "unknown option \"%s\"" % (o,)
1280
Doug Zongker85448772014-09-09 14:59:20 -07001281 if OPTIONS.search_path:
1282 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
1283 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -07001284
1285 return args
1286
1287
Tao Bao4c851b12016-09-19 13:54:38 -07001288def MakeTempFile(prefix='tmp', suffix=''):
Doug Zongkerfc44a512014-08-26 13:10:25 -07001289 """Make a temp file and add it to the list of things to be deleted
1290 when Cleanup() is called. Return the filename."""
1291 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix)
1292 os.close(fd)
1293 OPTIONS.tempfiles.append(fn)
1294 return fn
1295
1296
Tao Bao1c830bf2017-12-25 10:43:47 -08001297def MakeTempDir(prefix='tmp', suffix=''):
1298 """Makes a temporary dir that will be cleaned up with a call to Cleanup().
1299
1300 Returns:
1301 The absolute pathname of the new directory.
1302 """
1303 dir_name = tempfile.mkdtemp(suffix=suffix, prefix=prefix)
1304 OPTIONS.tempfiles.append(dir_name)
1305 return dir_name
1306
1307
Doug Zongkereef39442009-04-02 12:14:19 -07001308def Cleanup():
1309 for i in OPTIONS.tempfiles:
1310 if os.path.isdir(i):
Tao Bao1c830bf2017-12-25 10:43:47 -08001311 shutil.rmtree(i, ignore_errors=True)
Doug Zongkereef39442009-04-02 12:14:19 -07001312 else:
1313 os.remove(i)
Tao Bao1c830bf2017-12-25 10:43:47 -08001314 del OPTIONS.tempfiles[:]
Doug Zongker8ce7c252009-05-22 13:34:54 -07001315
1316
1317class PasswordManager(object):
1318 def __init__(self):
Tao Bao76def242017-11-21 09:25:31 -08001319 self.editor = os.getenv("EDITOR")
1320 self.pwfile = os.getenv("ANDROID_PW_FILE")
Doug Zongker8ce7c252009-05-22 13:34:54 -07001321
1322 def GetPasswords(self, items):
1323 """Get passwords corresponding to each string in 'items',
1324 returning a dict. (The dict may have keys in addition to the
1325 values in 'items'.)
1326
1327 Uses the passwords in $ANDROID_PW_FILE if available, letting the
1328 user edit that file to add more needed passwords. If no editor is
1329 available, or $ANDROID_PW_FILE isn't define, prompts the user
1330 interactively in the ordinary way.
1331 """
1332
1333 current = self.ReadFile()
1334
1335 first = True
1336 while True:
1337 missing = []
1338 for i in items:
1339 if i not in current or not current[i]:
1340 missing.append(i)
1341 # Are all the passwords already in the file?
Dan Albert8b72aef2015-03-23 19:13:21 -07001342 if not missing:
1343 return current
Doug Zongker8ce7c252009-05-22 13:34:54 -07001344
1345 for i in missing:
1346 current[i] = ""
1347
1348 if not first:
Tao Bao89fbb0f2017-01-10 10:47:58 -08001349 print("key file %s still missing some passwords." % (self.pwfile,))
Doug Zongker8ce7c252009-05-22 13:34:54 -07001350 answer = raw_input("try to edit again? [y]> ").strip()
1351 if answer and answer[0] not in 'yY':
1352 raise RuntimeError("key passwords unavailable")
1353 first = False
1354
1355 current = self.UpdateAndReadFile(current)
1356
Dan Albert8b72aef2015-03-23 19:13:21 -07001357 def PromptResult(self, current): # pylint: disable=no-self-use
Doug Zongker8ce7c252009-05-22 13:34:54 -07001358 """Prompt the user to enter a value (password) for each key in
1359 'current' whose value is fales. Returns a new dict with all the
1360 values.
1361 """
1362 result = {}
1363 for k, v in sorted(current.iteritems()):
1364 if v:
1365 result[k] = v
1366 else:
1367 while True:
Dan Albert8b72aef2015-03-23 19:13:21 -07001368 result[k] = getpass.getpass(
1369 "Enter password for %s key> " % k).strip()
1370 if result[k]:
1371 break
Doug Zongker8ce7c252009-05-22 13:34:54 -07001372 return result
1373
1374 def UpdateAndReadFile(self, current):
1375 if not self.editor or not self.pwfile:
1376 return self.PromptResult(current)
1377
1378 f = open(self.pwfile, "w")
Dan Albert8b72aef2015-03-23 19:13:21 -07001379 os.chmod(self.pwfile, 0o600)
Doug Zongker8ce7c252009-05-22 13:34:54 -07001380 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
1381 f.write("# (Additional spaces are harmless.)\n\n")
1382
1383 first_line = None
Dan Albert8b72aef2015-03-23 19:13:21 -07001384 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()])
1385 for i, (_, k, v) in enumerate(sorted_list):
Doug Zongker8ce7c252009-05-22 13:34:54 -07001386 f.write("[[[ %s ]]] %s\n" % (v, k))
1387 if not v and first_line is None:
1388 # position cursor on first line with no password.
1389 first_line = i + 4
1390 f.close()
1391
Tao Bao986ee862018-10-04 15:46:16 -07001392 RunAndCheckOutput([self.editor, "+%d" % (first_line,), self.pwfile])
Doug Zongker8ce7c252009-05-22 13:34:54 -07001393
1394 return self.ReadFile()
1395
1396 def ReadFile(self):
1397 result = {}
Dan Albert8b72aef2015-03-23 19:13:21 -07001398 if self.pwfile is None:
1399 return result
Doug Zongker8ce7c252009-05-22 13:34:54 -07001400 try:
1401 f = open(self.pwfile, "r")
1402 for line in f:
1403 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -07001404 if not line or line[0] == '#':
1405 continue
Doug Zongker8ce7c252009-05-22 13:34:54 -07001406 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
1407 if not m:
Tao Bao32fcdab2018-10-12 10:30:39 -07001408 logger.warning("Failed to parse password file: %s", line)
Doug Zongker8ce7c252009-05-22 13:34:54 -07001409 else:
1410 result[m.group(2)] = m.group(1)
1411 f.close()
Dan Albert8b72aef2015-03-23 19:13:21 -07001412 except IOError as e:
Doug Zongker8ce7c252009-05-22 13:34:54 -07001413 if e.errno != errno.ENOENT:
Tao Bao32fcdab2018-10-12 10:30:39 -07001414 logger.exception("Error reading password file:")
Doug Zongker8ce7c252009-05-22 13:34:54 -07001415 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -07001416
1417
Dan Albert8e0178d2015-01-27 15:53:15 -08001418def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
1419 compress_type=None):
1420 import datetime
1421
1422 # http://b/18015246
1423 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required
1424 # for files larger than 2GiB. We can work around this by adjusting their
1425 # limit. Note that `zipfile.writestr()` will not work for strings larger than
1426 # 2GiB. The Python interpreter sometimes rejects strings that large (though
1427 # it isn't clear to me exactly what circumstances cause this).
1428 # `zipfile.write()` must be used directly to work around this.
1429 #
1430 # This mess can be avoided if we port to python3.
1431 saved_zip64_limit = zipfile.ZIP64_LIMIT
1432 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1433
1434 if compress_type is None:
1435 compress_type = zip_file.compression
1436 if arcname is None:
1437 arcname = filename
1438
1439 saved_stat = os.stat(filename)
1440
1441 try:
1442 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the
1443 # file to be zipped and reset it when we're done.
1444 os.chmod(filename, perms)
1445
1446 # Use a fixed timestamp so the output is repeatable.
Bryan Henrye6d547d2018-07-31 18:32:00 -07001447 # Note: Use of fromtimestamp rather than utcfromtimestamp here is
1448 # intentional. zip stores datetimes in local time without a time zone
1449 # attached, so we need "epoch" but in the local time zone to get 2009/01/01
1450 # in the zip archive.
1451 local_epoch = datetime.datetime.fromtimestamp(0)
1452 timestamp = (datetime.datetime(2009, 1, 1) - local_epoch).total_seconds()
Dan Albert8e0178d2015-01-27 15:53:15 -08001453 os.utime(filename, (timestamp, timestamp))
1454
1455 zip_file.write(filename, arcname=arcname, compress_type=compress_type)
1456 finally:
1457 os.chmod(filename, saved_stat.st_mode)
1458 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime))
1459 zipfile.ZIP64_LIMIT = saved_zip64_limit
1460
1461
Tao Bao58c1b962015-05-20 09:32:18 -07001462def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
Tao Baof3282b42015-04-01 11:21:55 -07001463 compress_type=None):
1464 """Wrap zipfile.writestr() function to work around the zip64 limit.
1465
1466 Even with the ZIP64_LIMIT workaround, it won't allow writing a string
1467 longer than 2GiB. It gives 'OverflowError: size does not fit in an int'
1468 when calling crc32(bytes).
1469
1470 But it still works fine to write a shorter string into a large zip file.
1471 We should use ZipWrite() whenever possible, and only use ZipWriteStr()
1472 when we know the string won't be too long.
1473 """
1474
1475 saved_zip64_limit = zipfile.ZIP64_LIMIT
1476 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1477
1478 if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
1479 zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)
Dan Albert8b72aef2015-03-23 19:13:21 -07001480 zinfo.compress_type = zip_file.compression
Tao Bao58c1b962015-05-20 09:32:18 -07001481 if perms is None:
Tao Bao2a410582015-07-10 17:18:23 -07001482 perms = 0o100644
Geremy Condra36bd3652014-02-06 19:45:10 -08001483 else:
Tao Baof3282b42015-04-01 11:21:55 -07001484 zinfo = zinfo_or_arcname
1485
1486 # If compress_type is given, it overrides the value in zinfo.
1487 if compress_type is not None:
1488 zinfo.compress_type = compress_type
1489
Tao Bao58c1b962015-05-20 09:32:18 -07001490 # If perms is given, it has a priority.
1491 if perms is not None:
Tao Bao2a410582015-07-10 17:18:23 -07001492 # If perms doesn't set the file type, mark it as a regular file.
1493 if perms & 0o770000 == 0:
1494 perms |= 0o100000
Tao Bao58c1b962015-05-20 09:32:18 -07001495 zinfo.external_attr = perms << 16
1496
Tao Baof3282b42015-04-01 11:21:55 -07001497 # Use a fixed timestamp so the output is repeatable.
Tao Baof3282b42015-04-01 11:21:55 -07001498 zinfo.date_time = (2009, 1, 1, 0, 0, 0)
1499
Dan Albert8b72aef2015-03-23 19:13:21 -07001500 zip_file.writestr(zinfo, data)
Tao Baof3282b42015-04-01 11:21:55 -07001501 zipfile.ZIP64_LIMIT = saved_zip64_limit
1502
1503
Tao Bao89d7ab22017-12-14 17:05:33 -08001504def ZipDelete(zip_filename, entries):
1505 """Deletes entries from a ZIP file.
1506
1507 Since deleting entries from a ZIP file is not supported, it shells out to
1508 'zip -d'.
1509
1510 Args:
1511 zip_filename: The name of the ZIP file.
1512 entries: The name of the entry, or the list of names to be deleted.
1513
1514 Raises:
1515 AssertionError: In case of non-zero return from 'zip'.
1516 """
1517 if isinstance(entries, basestring):
1518 entries = [entries]
1519 cmd = ["zip", "-d", zip_filename] + entries
Tao Bao986ee862018-10-04 15:46:16 -07001520 RunAndCheckOutput(cmd)
Tao Bao89d7ab22017-12-14 17:05:33 -08001521
1522
Tao Baof3282b42015-04-01 11:21:55 -07001523def ZipClose(zip_file):
1524 # http://b/18015246
1525 # zipfile also refers to ZIP64_LIMIT during close() when it writes out the
1526 # central directory.
1527 saved_zip64_limit = zipfile.ZIP64_LIMIT
1528 zipfile.ZIP64_LIMIT = (1 << 32) - 1
1529
1530 zip_file.close()
1531
1532 zipfile.ZIP64_LIMIT = saved_zip64_limit
Doug Zongker05d3dea2009-06-22 11:32:31 -07001533
1534
1535class DeviceSpecificParams(object):
1536 module = None
1537 def __init__(self, **kwargs):
1538 """Keyword arguments to the constructor become attributes of this
1539 object, which is passed to all functions in the device-specific
1540 module."""
1541 for k, v in kwargs.iteritems():
1542 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -08001543 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -07001544
1545 if self.module is None:
1546 path = OPTIONS.device_specific
Dan Albert8b72aef2015-03-23 19:13:21 -07001547 if not path:
1548 return
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001549 try:
1550 if os.path.isdir(path):
1551 info = imp.find_module("releasetools", [path])
1552 else:
1553 d, f = os.path.split(path)
1554 b, x = os.path.splitext(f)
1555 if x == ".py":
1556 f = b
1557 info = imp.find_module(f, [d])
Tao Bao32fcdab2018-10-12 10:30:39 -07001558 logger.info("loaded device-specific extensions from %s", path)
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001559 self.module = imp.load_module("device_specific", *info)
1560 except ImportError:
Tao Bao32fcdab2018-10-12 10:30:39 -07001561 logger.info("unable to load device-specific module; assuming none")
Doug Zongker05d3dea2009-06-22 11:32:31 -07001562
1563 def _DoCall(self, function_name, *args, **kwargs):
1564 """Call the named function in the device-specific module, passing
1565 the given args and kwargs. The first argument to the call will be
1566 the DeviceSpecific object itself. If there is no module, or the
1567 module does not define the function, return the value of the
1568 'default' kwarg (which itself defaults to None)."""
1569 if self.module is None or not hasattr(self.module, function_name):
Tao Bao76def242017-11-21 09:25:31 -08001570 return kwargs.get("default")
Doug Zongker05d3dea2009-06-22 11:32:31 -07001571 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
1572
1573 def FullOTA_Assertions(self):
1574 """Called after emitting the block of assertions at the top of a
1575 full OTA package. Implementations can add whatever additional
1576 assertions they like."""
1577 return self._DoCall("FullOTA_Assertions")
1578
Doug Zongkere5ff5902012-01-17 10:55:37 -08001579 def FullOTA_InstallBegin(self):
1580 """Called at the start of full OTA installation."""
1581 return self._DoCall("FullOTA_InstallBegin")
1582
Yifan Hong10c530d2018-12-27 17:34:18 -08001583 def FullOTA_GetBlockDifferences(self):
1584 """Called during full OTA installation and verification.
1585 Implementation should return a list of BlockDifference objects describing
1586 the update on each additional partitions.
1587 """
1588 return self._DoCall("FullOTA_GetBlockDifferences")
1589
Doug Zongker05d3dea2009-06-22 11:32:31 -07001590 def FullOTA_InstallEnd(self):
1591 """Called at the end of full OTA installation; typically this is
1592 used to install the image for the device's baseband processor."""
1593 return self._DoCall("FullOTA_InstallEnd")
1594
1595 def IncrementalOTA_Assertions(self):
1596 """Called after emitting the block of assertions at the top of an
1597 incremental OTA package. Implementations can add whatever
1598 additional assertions they like."""
1599 return self._DoCall("IncrementalOTA_Assertions")
1600
Doug Zongkere5ff5902012-01-17 10:55:37 -08001601 def IncrementalOTA_VerifyBegin(self):
1602 """Called at the start of the verification phase of incremental
1603 OTA installation; additional checks can be placed here to abort
1604 the script before any changes are made."""
1605 return self._DoCall("IncrementalOTA_VerifyBegin")
1606
Doug Zongker05d3dea2009-06-22 11:32:31 -07001607 def IncrementalOTA_VerifyEnd(self):
1608 """Called at the end of the verification phase of incremental OTA
1609 installation; additional checks can be placed here to abort the
1610 script before any changes are made."""
1611 return self._DoCall("IncrementalOTA_VerifyEnd")
1612
Doug Zongkere5ff5902012-01-17 10:55:37 -08001613 def IncrementalOTA_InstallBegin(self):
1614 """Called at the start of incremental OTA installation (after
1615 verification is complete)."""
1616 return self._DoCall("IncrementalOTA_InstallBegin")
1617
Yifan Hong10c530d2018-12-27 17:34:18 -08001618 def IncrementalOTA_GetBlockDifferences(self):
1619 """Called during incremental OTA installation and verification.
1620 Implementation should return a list of BlockDifference objects describing
1621 the update on each additional partitions.
1622 """
1623 return self._DoCall("IncrementalOTA_GetBlockDifferences")
1624
Doug Zongker05d3dea2009-06-22 11:32:31 -07001625 def IncrementalOTA_InstallEnd(self):
1626 """Called at the end of incremental OTA installation; typically
1627 this is used to install the image for the device's baseband
1628 processor."""
1629 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001630
Tao Bao9bc6bb22015-11-09 16:58:28 -08001631 def VerifyOTA_Assertions(self):
1632 return self._DoCall("VerifyOTA_Assertions")
1633
Tao Bao76def242017-11-21 09:25:31 -08001634
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001635class File(object):
Tao Bao76def242017-11-21 09:25:31 -08001636 def __init__(self, name, data, compress_size=None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001637 self.name = name
1638 self.data = data
1639 self.size = len(data)
YOUNG HO CHAccc5c402016-10-13 13:40:46 +09001640 self.compress_size = compress_size or self.size
Doug Zongker55d93282011-01-25 17:03:34 -08001641 self.sha1 = sha1(data).hexdigest()
1642
1643 @classmethod
1644 def FromLocalFile(cls, name, diskname):
1645 f = open(diskname, "rb")
1646 data = f.read()
1647 f.close()
1648 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001649
1650 def WriteToTemp(self):
1651 t = tempfile.NamedTemporaryFile()
1652 t.write(self.data)
1653 t.flush()
1654 return t
1655
Dan Willemsen2ee00d52017-03-05 19:51:56 -08001656 def WriteToDir(self, d):
1657 with open(os.path.join(d, self.name), "wb") as fp:
1658 fp.write(self.data)
1659
Geremy Condra36bd3652014-02-06 19:45:10 -08001660 def AddToZip(self, z, compression=None):
Tao Baof3282b42015-04-01 11:21:55 -07001661 ZipWriteStr(z, self.name, self.data, compress_type=compression)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001662
Tao Bao76def242017-11-21 09:25:31 -08001663
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001664DIFF_PROGRAM_BY_EXT = {
1665 ".gz" : "imgdiff",
1666 ".zip" : ["imgdiff", "-z"],
1667 ".jar" : ["imgdiff", "-z"],
1668 ".apk" : ["imgdiff", "-z"],
1669 ".img" : "imgdiff",
1670 }
1671
Tao Bao76def242017-11-21 09:25:31 -08001672
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001673class Difference(object):
Doug Zongker24cd2802012-08-14 16:36:15 -07001674 def __init__(self, tf, sf, diff_program=None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001675 self.tf = tf
1676 self.sf = sf
1677 self.patch = None
Doug Zongker24cd2802012-08-14 16:36:15 -07001678 self.diff_program = diff_program
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001679
1680 def ComputePatch(self):
1681 """Compute the patch (as a string of data) needed to turn sf into
1682 tf. Returns the same tuple as GetPatch()."""
1683
1684 tf = self.tf
1685 sf = self.sf
1686
Doug Zongker24cd2802012-08-14 16:36:15 -07001687 if self.diff_program:
1688 diff_program = self.diff_program
1689 else:
1690 ext = os.path.splitext(tf.name)[1]
1691 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001692
1693 ttemp = tf.WriteToTemp()
1694 stemp = sf.WriteToTemp()
1695
1696 ext = os.path.splitext(tf.name)[1]
1697
1698 try:
1699 ptemp = tempfile.NamedTemporaryFile()
1700 if isinstance(diff_program, list):
1701 cmd = copy.copy(diff_program)
1702 else:
1703 cmd = [diff_program]
1704 cmd.append(stemp.name)
1705 cmd.append(ttemp.name)
1706 cmd.append(ptemp.name)
1707 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Doug Zongkerf8340082014-08-05 10:39:37 -07001708 err = []
1709 def run():
1710 _, e = p.communicate()
Dan Albert8b72aef2015-03-23 19:13:21 -07001711 if e:
1712 err.append(e)
Doug Zongkerf8340082014-08-05 10:39:37 -07001713 th = threading.Thread(target=run)
1714 th.start()
1715 th.join(timeout=300) # 5 mins
1716 if th.is_alive():
Tao Bao32fcdab2018-10-12 10:30:39 -07001717 logger.warning("diff command timed out")
Doug Zongkerf8340082014-08-05 10:39:37 -07001718 p.terminate()
1719 th.join(5)
1720 if th.is_alive():
1721 p.kill()
1722 th.join()
1723
Tianjie Xua2a9f992018-01-05 15:15:54 -08001724 if p.returncode != 0:
Tao Bao32fcdab2018-10-12 10:30:39 -07001725 logger.warning("Failure running %s:\n%s\n", diff_program, "".join(err))
Doug Zongkerf8340082014-08-05 10:39:37 -07001726 self.patch = None
1727 return None, None, None
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001728 diff = ptemp.read()
1729 finally:
1730 ptemp.close()
1731 stemp.close()
1732 ttemp.close()
1733
1734 self.patch = diff
1735 return self.tf, self.sf, self.patch
1736
1737
1738 def GetPatch(self):
Tao Bao76def242017-11-21 09:25:31 -08001739 """Returns a tuple of (target_file, source_file, patch_data).
1740
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001741 patch_data may be None if ComputePatch hasn't been called, or if
Tao Bao76def242017-11-21 09:25:31 -08001742 computing the patch failed.
1743 """
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001744 return self.tf, self.sf, self.patch
1745
1746
1747def ComputeDifferences(diffs):
1748 """Call ComputePatch on all the Difference objects in 'diffs'."""
Tao Bao32fcdab2018-10-12 10:30:39 -07001749 logger.info("%d diffs to compute", len(diffs))
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001750
1751 # Do the largest files first, to try and reduce the long-pole effect.
1752 by_size = [(i.tf.size, i) for i in diffs]
1753 by_size.sort(reverse=True)
1754 by_size = [i[1] for i in by_size]
1755
1756 lock = threading.Lock()
1757 diff_iter = iter(by_size) # accessed under lock
1758
1759 def worker():
1760 try:
1761 lock.acquire()
1762 for d in diff_iter:
1763 lock.release()
1764 start = time.time()
1765 d.ComputePatch()
1766 dur = time.time() - start
1767 lock.acquire()
1768
1769 tf, sf, patch = d.GetPatch()
1770 if sf.name == tf.name:
1771 name = tf.name
1772 else:
1773 name = "%s (%s)" % (tf.name, sf.name)
1774 if patch is None:
Tao Bao32fcdab2018-10-12 10:30:39 -07001775 logger.error("patching failed! %40s", name)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001776 else:
Tao Bao32fcdab2018-10-12 10:30:39 -07001777 logger.info(
1778 "%8.2f sec %8d / %8d bytes (%6.2f%%) %s", dur, len(patch),
1779 tf.size, 100.0 * len(patch) / tf.size, name)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001780 lock.release()
Tao Bao32fcdab2018-10-12 10:30:39 -07001781 except Exception:
1782 logger.exception("Failed to compute diff from worker")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001783 raise
1784
1785 # start worker threads; wait for them all to finish.
1786 threads = [threading.Thread(target=worker)
1787 for i in range(OPTIONS.worker_threads)]
1788 for th in threads:
1789 th.start()
1790 while threads:
1791 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -07001792
1793
Dan Albert8b72aef2015-03-23 19:13:21 -07001794class BlockDifference(object):
1795 def __init__(self, partition, tgt, src=None, check_first_block=False,
Tao Bao293fd132016-06-11 12:19:23 -07001796 version=None, disable_imgdiff=False):
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001797 self.tgt = tgt
1798 self.src = src
1799 self.partition = partition
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001800 self.check_first_block = check_first_block
Tao Bao293fd132016-06-11 12:19:23 -07001801 self.disable_imgdiff = disable_imgdiff
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001802
Tao Baodd2a5892015-03-12 12:32:37 -07001803 if version is None:
Tao Bao0582cb62017-12-21 11:47:01 -08001804 version = max(
1805 int(i) for i in
1806 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(","))
Tao Bao8fad03e2017-03-01 14:36:26 -08001807 assert version >= 3
Tao Baodd2a5892015-03-12 12:32:37 -07001808 self.version = version
Doug Zongker62338182014-09-08 08:29:55 -07001809
1810 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads,
Tao Bao293fd132016-06-11 12:19:23 -07001811 version=self.version,
1812 disable_imgdiff=self.disable_imgdiff)
Tao Bao04bce3a2018-02-28 11:11:00 -08001813 self.path = os.path.join(MakeTempDir(), partition)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001814 b.Compute(self.path)
Tao Baod8d14be2016-02-04 14:26:02 -08001815 self._required_cache = b.max_stashed_size
Tao Baod522bdc2016-04-12 15:53:16 -07001816 self.touched_src_ranges = b.touched_src_ranges
1817 self.touched_src_sha1 = b.touched_src_sha1
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001818
Yifan Hong10c530d2018-12-27 17:34:18 -08001819 # On devices with dynamic partitions, for new partitions,
1820 # src is None but OPTIONS.source_info_dict is not.
1821 if OPTIONS.source_info_dict is None:
1822 is_dynamic_build = OPTIONS.info_dict.get(
1823 "use_dynamic_partitions") == "true"
Yifan Hongbb2658d2019-01-25 12:30:58 -08001824 is_dynamic_source = False
Tao Baoaac4ad52015-10-16 15:26:34 -07001825 else:
Yifan Hong10c530d2018-12-27 17:34:18 -08001826 is_dynamic_build = OPTIONS.source_info_dict.get(
1827 "use_dynamic_partitions") == "true"
Yifan Hongbb2658d2019-01-25 12:30:58 -08001828 is_dynamic_source = partition in shlex.split(
1829 OPTIONS.source_info_dict.get("dynamic_partition_list", "").strip())
Yifan Hong10c530d2018-12-27 17:34:18 -08001830
Yifan Hongbb2658d2019-01-25 12:30:58 -08001831 is_dynamic_target = partition in shlex.split(
Yifan Hong10c530d2018-12-27 17:34:18 -08001832 OPTIONS.info_dict.get("dynamic_partition_list", "").strip())
1833
Yifan Hongbb2658d2019-01-25 12:30:58 -08001834 # For dynamic partitions builds, check partition list in both source
1835 # and target build because new partitions may be added, and existing
1836 # partitions may be removed.
1837 is_dynamic = is_dynamic_build and (is_dynamic_source or is_dynamic_target)
1838
Yifan Hong10c530d2018-12-27 17:34:18 -08001839 if is_dynamic:
1840 self.device = 'map_partition("%s")' % partition
1841 else:
1842 if OPTIONS.source_info_dict is None:
1843 _, device_path = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)
1844 else:
1845 _, device_path = GetTypeAndDevice("/" + partition,
1846 OPTIONS.source_info_dict)
1847 self.device = '"%s"' % device_path
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001848
Tao Baod8d14be2016-02-04 14:26:02 -08001849 @property
1850 def required_cache(self):
1851 return self._required_cache
1852
Tao Bao76def242017-11-21 09:25:31 -08001853 def WriteScript(self, script, output_zip, progress=None,
1854 write_verify_script=False):
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001855 if not self.src:
1856 # write the output unconditionally
Jesse Zhao75bcea02015-01-06 10:59:53 -08001857 script.Print("Patching %s image unconditionally..." % (self.partition,))
1858 else:
1859 script.Print("Patching %s image after verification." % (self.partition,))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001860
Dan Albert8b72aef2015-03-23 19:13:21 -07001861 if progress:
1862 script.ShowProgress(progress, 0)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001863 self._WriteUpdate(script, output_zip)
Tao Bao76def242017-11-21 09:25:31 -08001864
1865 if write_verify_script:
Yifan Hong10c530d2018-12-27 17:34:18 -08001866 self.WritePostInstallVerifyScript(script)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001867
Tao Bao9bc6bb22015-11-09 16:58:28 -08001868 def WriteStrictVerifyScript(self, script):
1869 """Verify all the blocks in the care_map, including clobbered blocks.
1870
1871 This differs from the WriteVerifyScript() function: a) it prints different
1872 error messages; b) it doesn't allow half-way updated images to pass the
1873 verification."""
1874
1875 partition = self.partition
1876 script.Print("Verifying %s..." % (partition,))
1877 ranges = self.tgt.care_map
1878 ranges_str = ranges.to_string_raw()
Tao Bao76def242017-11-21 09:25:31 -08001879 script.AppendExtra(
Yifan Hong10c530d2018-12-27 17:34:18 -08001880 'range_sha1(%s, "%s") == "%s" && ui_print(" Verified.") || '
1881 'ui_print("%s has unexpected contents.");' % (
Tao Bao76def242017-11-21 09:25:31 -08001882 self.device, ranges_str,
1883 self.tgt.TotalSha1(include_clobbered_blocks=True),
Yifan Hong10c530d2018-12-27 17:34:18 -08001884 self.partition))
Tao Bao9bc6bb22015-11-09 16:58:28 -08001885 script.AppendExtra("")
1886
Tao Baod522bdc2016-04-12 15:53:16 -07001887 def WriteVerifyScript(self, script, touched_blocks_only=False):
Sami Tolvanendd67a292014-12-09 16:40:34 +00001888 partition = self.partition
Tao Baof9efe282016-04-14 15:58:05 -07001889
1890 # full OTA
Jesse Zhao75bcea02015-01-06 10:59:53 -08001891 if not self.src:
Sami Tolvanendd67a292014-12-09 16:40:34 +00001892 script.Print("Image %s will be patched unconditionally." % (partition,))
Tao Baof9efe282016-04-14 15:58:05 -07001893
1894 # incremental OTA
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001895 else:
Tao Bao8fad03e2017-03-01 14:36:26 -08001896 if touched_blocks_only:
Tao Baod522bdc2016-04-12 15:53:16 -07001897 ranges = self.touched_src_ranges
1898 expected_sha1 = self.touched_src_sha1
1899 else:
1900 ranges = self.src.care_map.subtract(self.src.clobbered_blocks)
1901 expected_sha1 = self.src.TotalSha1()
Tao Baof9efe282016-04-14 15:58:05 -07001902
1903 # No blocks to be checked, skipping.
1904 if not ranges:
1905 return
1906
Tao Bao5ece99d2015-05-12 11:42:31 -07001907 ranges_str = ranges.to_string_raw()
Tao Bao76def242017-11-21 09:25:31 -08001908 script.AppendExtra(
Yifan Hong10c530d2018-12-27 17:34:18 -08001909 'if (range_sha1(%s, "%s") == "%s" || block_image_verify(%s, '
Tao Bao76def242017-11-21 09:25:31 -08001910 'package_extract_file("%s.transfer.list"), "%s.new.dat", '
1911 '"%s.patch.dat")) then' % (
1912 self.device, ranges_str, expected_sha1,
1913 self.device, partition, partition, partition))
Tao Baodd2a5892015-03-12 12:32:37 -07001914 script.Print('Verified %s image...' % (partition,))
Dan Albert8b72aef2015-03-23 19:13:21 -07001915 script.AppendExtra('else')
Sami Tolvanendd67a292014-12-09 16:40:34 +00001916
Tianjie Xufc3422a2015-12-15 11:53:59 -08001917 if self.version >= 4:
1918
1919 # Bug: 21124327
1920 # When generating incrementals for the system and vendor partitions in
1921 # version 4 or newer, explicitly check the first block (which contains
1922 # the superblock) of the partition to see if it's what we expect. If
1923 # this check fails, give an explicit log message about the partition
1924 # having been remounted R/W (the most likely explanation).
1925 if self.check_first_block:
Yifan Hong10c530d2018-12-27 17:34:18 -08001926 script.AppendExtra('check_first_block(%s);' % (self.device,))
Tianjie Xufc3422a2015-12-15 11:53:59 -08001927
1928 # If version >= 4, try block recovery before abort update
Tianjie Xu209db462016-05-24 17:34:52 -07001929 if partition == "system":
1930 code = ErrorCode.SYSTEM_RECOVER_FAILURE
1931 else:
1932 code = ErrorCode.VENDOR_RECOVER_FAILURE
Tianjie Xufc3422a2015-12-15 11:53:59 -08001933 script.AppendExtra((
Yifan Hong10c530d2018-12-27 17:34:18 -08001934 'ifelse (block_image_recover({device}, "{ranges}") && '
1935 'block_image_verify({device}, '
Tianjie Xufc3422a2015-12-15 11:53:59 -08001936 'package_extract_file("{partition}.transfer.list"), '
1937 '"{partition}.new.dat", "{partition}.patch.dat"), '
1938 'ui_print("{partition} recovered successfully."), '
Tianjie Xu209db462016-05-24 17:34:52 -07001939 'abort("E{code}: {partition} partition fails to recover"));\n'
Tianjie Xufc3422a2015-12-15 11:53:59 -08001940 'endif;').format(device=self.device, ranges=ranges_str,
Tianjie Xu209db462016-05-24 17:34:52 -07001941 partition=partition, code=code))
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001942
Tao Baodd2a5892015-03-12 12:32:37 -07001943 # Abort the OTA update. Note that the incremental OTA cannot be applied
1944 # even if it may match the checksum of the target partition.
1945 # a) If version < 3, operations like move and erase will make changes
1946 # unconditionally and damage the partition.
1947 # b) If version >= 3, it won't even reach here.
Tianjie Xufc3422a2015-12-15 11:53:59 -08001948 else:
Tianjie Xu209db462016-05-24 17:34:52 -07001949 if partition == "system":
1950 code = ErrorCode.SYSTEM_VERIFICATION_FAILURE
1951 else:
1952 code = ErrorCode.VENDOR_VERIFICATION_FAILURE
1953 script.AppendExtra((
1954 'abort("E%d: %s partition has unexpected contents");\n'
1955 'endif;') % (code, partition))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001956
Yifan Hong10c530d2018-12-27 17:34:18 -08001957 def WritePostInstallVerifyScript(self, script):
Tao Bao5fcaaef2015-06-01 13:40:49 -07001958 partition = self.partition
1959 script.Print('Verifying the updated %s image...' % (partition,))
1960 # Unlike pre-install verification, clobbered_blocks should not be ignored.
1961 ranges = self.tgt.care_map
1962 ranges_str = ranges.to_string_raw()
Tao Bao76def242017-11-21 09:25:31 -08001963 script.AppendExtra(
Yifan Hong10c530d2018-12-27 17:34:18 -08001964 'if range_sha1(%s, "%s") == "%s" then' % (
Tao Bao76def242017-11-21 09:25:31 -08001965 self.device, ranges_str,
1966 self.tgt.TotalSha1(include_clobbered_blocks=True)))
Tao Baoe9b61912015-07-09 17:37:49 -07001967
1968 # Bug: 20881595
1969 # Verify that extended blocks are really zeroed out.
1970 if self.tgt.extended:
1971 ranges_str = self.tgt.extended.to_string_raw()
Tao Bao76def242017-11-21 09:25:31 -08001972 script.AppendExtra(
Yifan Hong10c530d2018-12-27 17:34:18 -08001973 'if range_sha1(%s, "%s") == "%s" then' % (
Tao Bao76def242017-11-21 09:25:31 -08001974 self.device, ranges_str,
1975 self._HashZeroBlocks(self.tgt.extended.size())))
Tao Baoe9b61912015-07-09 17:37:49 -07001976 script.Print('Verified the updated %s image.' % (partition,))
Tianjie Xu209db462016-05-24 17:34:52 -07001977 if partition == "system":
1978 code = ErrorCode.SYSTEM_NONZERO_CONTENTS
1979 else:
1980 code = ErrorCode.VENDOR_NONZERO_CONTENTS
Tao Baoe9b61912015-07-09 17:37:49 -07001981 script.AppendExtra(
1982 'else\n'
Tianjie Xu209db462016-05-24 17:34:52 -07001983 ' abort("E%d: %s partition has unexpected non-zero contents after '
1984 'OTA update");\n'
1985 'endif;' % (code, partition))
Tao Baoe9b61912015-07-09 17:37:49 -07001986 else:
1987 script.Print('Verified the updated %s image.' % (partition,))
1988
Tianjie Xu209db462016-05-24 17:34:52 -07001989 if partition == "system":
1990 code = ErrorCode.SYSTEM_UNEXPECTED_CONTENTS
1991 else:
1992 code = ErrorCode.VENDOR_UNEXPECTED_CONTENTS
1993
Tao Bao5fcaaef2015-06-01 13:40:49 -07001994 script.AppendExtra(
1995 'else\n'
Tianjie Xu209db462016-05-24 17:34:52 -07001996 ' abort("E%d: %s partition has unexpected contents after OTA '
1997 'update");\n'
1998 'endif;' % (code, partition))
Tao Bao5fcaaef2015-06-01 13:40:49 -07001999
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07002000 def _WriteUpdate(self, script, output_zip):
Dan Albert8e0178d2015-01-27 15:53:15 -08002001 ZipWrite(output_zip,
2002 '{}.transfer.list'.format(self.path),
2003 '{}.transfer.list'.format(self.partition))
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002004
Tao Bao76def242017-11-21 09:25:31 -08002005 # For full OTA, compress the new.dat with brotli with quality 6 to reduce
2006 # its size. Quailty 9 almost triples the compression time but doesn't
2007 # further reduce the size too much. For a typical 1.8G system.new.dat
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002008 # zip | brotli(quality 6) | brotli(quality 9)
2009 # compressed_size: 942M | 869M (~8% reduced) | 854M
2010 # compression_time: 75s | 265s | 719s
2011 # decompression_time: 15s | 25s | 25s
2012
2013 if not self.src:
Alex Deymob10e07a2017-11-09 23:53:42 +01002014 brotli_cmd = ['brotli', '--quality=6',
2015 '--output={}.new.dat.br'.format(self.path),
2016 '{}.new.dat'.format(self.path)]
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002017 print("Compressing {}.new.dat with brotli".format(self.partition))
Tao Bao986ee862018-10-04 15:46:16 -07002018 RunAndCheckOutput(brotli_cmd)
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002019
2020 new_data_name = '{}.new.dat.br'.format(self.partition)
2021 ZipWrite(output_zip,
2022 '{}.new.dat.br'.format(self.path),
2023 new_data_name,
2024 compress_type=zipfile.ZIP_STORED)
2025 else:
2026 new_data_name = '{}.new.dat'.format(self.partition)
2027 ZipWrite(output_zip, '{}.new.dat'.format(self.path), new_data_name)
2028
Dan Albert8e0178d2015-01-27 15:53:15 -08002029 ZipWrite(output_zip,
2030 '{}.patch.dat'.format(self.path),
2031 '{}.patch.dat'.format(self.partition),
2032 compress_type=zipfile.ZIP_STORED)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07002033
Tianjie Xu209db462016-05-24 17:34:52 -07002034 if self.partition == "system":
2035 code = ErrorCode.SYSTEM_UPDATE_FAILURE
2036 else:
2037 code = ErrorCode.VENDOR_UPDATE_FAILURE
2038
Yifan Hong10c530d2018-12-27 17:34:18 -08002039 call = ('block_image_update({device}, '
Dan Albert8e0178d2015-01-27 15:53:15 -08002040 'package_extract_file("{partition}.transfer.list"), '
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002041 '"{new_data_name}", "{partition}.patch.dat") ||\n'
Tianjie Xu209db462016-05-24 17:34:52 -07002042 ' abort("E{code}: Failed to update {partition} image.");'.format(
Tianjie Xub0a29ad2017-07-06 15:13:59 -07002043 device=self.device, partition=self.partition,
2044 new_data_name=new_data_name, code=code))
Dan Albert8b72aef2015-03-23 19:13:21 -07002045 script.AppendExtra(script.WordWrap(call))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07002046
Dan Albert8b72aef2015-03-23 19:13:21 -07002047 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use
Sami Tolvanendd67a292014-12-09 16:40:34 +00002048 data = source.ReadRangeSet(ranges)
2049 ctx = sha1()
2050
2051 for p in data:
2052 ctx.update(p)
2053
2054 return ctx.hexdigest()
2055
Tao Baoe9b61912015-07-09 17:37:49 -07002056 def _HashZeroBlocks(self, num_blocks): # pylint: disable=no-self-use
2057 """Return the hash value for all zero blocks."""
2058 zero_block = '\x00' * 4096
2059 ctx = sha1()
2060 for _ in range(num_blocks):
2061 ctx.update(zero_block)
2062
2063 return ctx.hexdigest()
2064
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07002065
2066DataImage = blockimgdiff.DataImage
2067
Tao Bao76def242017-11-21 09:25:31 -08002068
Doug Zongker96a57e72010-09-26 14:57:41 -07002069# map recovery.fstab's fs_types to mount/format "partition types"
Dan Albert8b72aef2015-03-23 19:13:21 -07002070PARTITION_TYPES = {
Dan Albert8b72aef2015-03-23 19:13:21 -07002071 "ext4": "EMMC",
2072 "emmc": "EMMC",
Mohamad Ayyash95e74c12015-05-01 15:39:36 -07002073 "f2fs": "EMMC",
2074 "squashfs": "EMMC"
Dan Albert8b72aef2015-03-23 19:13:21 -07002075}
Doug Zongker96a57e72010-09-26 14:57:41 -07002076
Tao Bao76def242017-11-21 09:25:31 -08002077
Doug Zongker96a57e72010-09-26 14:57:41 -07002078def GetTypeAndDevice(mount_point, info):
2079 fstab = info["fstab"]
2080 if fstab:
Dan Albert8b72aef2015-03-23 19:13:21 -07002081 return (PARTITION_TYPES[fstab[mount_point].fs_type],
2082 fstab[mount_point].device)
Doug Zongker96a57e72010-09-26 14:57:41 -07002083 else:
Dan Albert8b72aef2015-03-23 19:13:21 -07002084 raise KeyError
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00002085
2086
2087def ParseCertificate(data):
Tao Bao17e4e612018-02-16 17:12:54 -08002088 """Parses and converts a PEM-encoded certificate into DER-encoded.
2089
2090 This gives the same result as `openssl x509 -in <filename> -outform DER`.
2091
2092 Returns:
2093 The decoded certificate string.
2094 """
2095 cert_buffer = []
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00002096 save = False
2097 for line in data.split("\n"):
2098 if "--END CERTIFICATE--" in line:
2099 break
2100 if save:
Tao Bao17e4e612018-02-16 17:12:54 -08002101 cert_buffer.append(line)
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00002102 if "--BEGIN CERTIFICATE--" in line:
2103 save = True
Tao Bao17e4e612018-02-16 17:12:54 -08002104 cert = "".join(cert_buffer).decode('base64')
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00002105 return cert
Doug Zongkerc9253822014-02-04 12:17:58 -08002106
Tao Bao04e1f012018-02-04 12:13:35 -08002107
2108def ExtractPublicKey(cert):
2109 """Extracts the public key (PEM-encoded) from the given certificate file.
2110
2111 Args:
2112 cert: The certificate filename.
2113
2114 Returns:
2115 The public key string.
2116
2117 Raises:
2118 AssertionError: On non-zero return from 'openssl'.
2119 """
2120 # The behavior with '-out' is different between openssl 1.1 and openssl 1.0.
2121 # While openssl 1.1 writes the key into the given filename followed by '-out',
2122 # openssl 1.0 (both of 1.0.1 and 1.0.2) doesn't. So we collect the output from
2123 # stdout instead.
2124 cmd = ['openssl', 'x509', '-pubkey', '-noout', '-in', cert]
2125 proc = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
2126 pubkey, stderrdata = proc.communicate()
2127 assert proc.returncode == 0, \
2128 'Failed to dump public key from certificate: %s\n%s' % (cert, stderrdata)
2129 return pubkey
2130
2131
Tao Bao2cc0ca12019-03-15 10:44:43 -07002132def ExtractAvbPublicKey(key):
2133 """Extracts the AVB public key from the given public or private key.
2134
2135 Args:
2136 key: The input key file, which should be PEM-encoded public or private key.
2137
2138 Returns:
2139 The path to the extracted AVB public key file.
2140 """
2141 output = MakeTempFile(prefix='avb-', suffix='.avbpubkey')
2142 RunAndCheckOutput(
2143 ['avbtool', 'extract_public_key', "--key", key, "--output", output])
2144 return output
2145
2146
Doug Zongker412c02f2014-02-13 10:58:24 -08002147def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img,
2148 info_dict=None):
Tao Bao6d5d6232018-03-09 17:04:42 -08002149 """Generates the recovery-from-boot patch and writes the script to output.
Doug Zongkerc9253822014-02-04 12:17:58 -08002150
Tao Bao6d5d6232018-03-09 17:04:42 -08002151 Most of the space in the boot and recovery images is just the kernel, which is
2152 identical for the two, so the resulting patch should be efficient. Add it to
2153 the output zip, along with a shell script that is run from init.rc on first
2154 boot to actually do the patching and install the new recovery image.
2155
2156 Args:
2157 input_dir: The top-level input directory of the target-files.zip.
2158 output_sink: The callback function that writes the result.
2159 recovery_img: File object for the recovery image.
2160 boot_img: File objects for the boot image.
2161 info_dict: A dict returned by common.LoadInfoDict() on the input
2162 target_files. Will use OPTIONS.info_dict if None has been given.
Doug Zongkerc9253822014-02-04 12:17:58 -08002163 """
Doug Zongker412c02f2014-02-13 10:58:24 -08002164 if info_dict is None:
2165 info_dict = OPTIONS.info_dict
2166
Tao Bao6d5d6232018-03-09 17:04:42 -08002167 full_recovery_image = info_dict.get("full_recovery_image") == "true"
Doug Zongkerc9253822014-02-04 12:17:58 -08002168
Tao Baof2cffbd2015-07-22 12:33:18 -07002169 if full_recovery_image:
2170 output_sink("etc/recovery.img", recovery_img.data)
2171
2172 else:
Tao Bao6d5d6232018-03-09 17:04:42 -08002173 system_root_image = info_dict.get("system_root_image") == "true"
Tao Baof2cffbd2015-07-22 12:33:18 -07002174 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat")
Tao Bao6d5d6232018-03-09 17:04:42 -08002175 # With system-root-image, boot and recovery images will have mismatching
2176 # entries (only recovery has the ramdisk entry) (Bug: 72731506). Use bsdiff
2177 # to handle such a case.
2178 if system_root_image:
2179 diff_program = ["bsdiff"]
Tao Baof2cffbd2015-07-22 12:33:18 -07002180 bonus_args = ""
Tao Bao6d5d6232018-03-09 17:04:42 -08002181 assert not os.path.exists(path)
2182 else:
2183 diff_program = ["imgdiff"]
2184 if os.path.exists(path):
2185 diff_program.append("-b")
2186 diff_program.append(path)
Tao Bao4948aed2018-07-13 16:11:16 -07002187 bonus_args = "--bonus /system/etc/recovery-resource.dat"
Tao Bao6d5d6232018-03-09 17:04:42 -08002188 else:
2189 bonus_args = ""
Tao Baof2cffbd2015-07-22 12:33:18 -07002190
2191 d = Difference(recovery_img, boot_img, diff_program=diff_program)
2192 _, _, patch = d.ComputePatch()
2193 output_sink("recovery-from-boot.p", patch)
Doug Zongkerc9253822014-02-04 12:17:58 -08002194
Dan Albertebb19aa2015-03-27 19:11:53 -07002195 try:
Tao Bao6f0b2192015-10-13 16:37:12 -07002196 # The following GetTypeAndDevice()s need to use the path in the target
2197 # info_dict instead of source_info_dict.
Dan Albertebb19aa2015-03-27 19:11:53 -07002198 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict)
2199 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict)
2200 except KeyError:
Ying Wanga961a092014-07-29 11:42:37 -07002201 return
Doug Zongkerc9253822014-02-04 12:17:58 -08002202
Tao Baof2cffbd2015-07-22 12:33:18 -07002203 if full_recovery_image:
2204 sh = """#!/system/bin/sh
Tao Bao4948aed2018-07-13 16:11:16 -07002205if ! applypatch --check %(type)s:%(device)s:%(size)d:%(sha1)s; then
2206 applypatch \\
2207 --flash /system/etc/recovery.img \\
2208 --target %(type)s:%(device)s:%(size)d:%(sha1)s && \\
2209 log -t recovery "Installing new recovery image: succeeded" || \\
2210 log -t recovery "Installing new recovery image: failed"
Tao Baof2cffbd2015-07-22 12:33:18 -07002211else
2212 log -t recovery "Recovery image already installed"
2213fi
2214""" % {'type': recovery_type,
2215 'device': recovery_device,
2216 'sha1': recovery_img.sha1,
2217 'size': recovery_img.size}
2218 else:
2219 sh = """#!/system/bin/sh
Tao Bao4948aed2018-07-13 16:11:16 -07002220if ! applypatch --check %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then
2221 applypatch %(bonus_args)s \\
2222 --patch /system/recovery-from-boot.p \\
2223 --source %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s \\
2224 --target %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s && \\
2225 log -t recovery "Installing new recovery image: succeeded" || \\
2226 log -t recovery "Installing new recovery image: failed"
Doug Zongkerc9253822014-02-04 12:17:58 -08002227else
2228 log -t recovery "Recovery image already installed"
2229fi
Dan Albert8b72aef2015-03-23 19:13:21 -07002230""" % {'boot_size': boot_img.size,
2231 'boot_sha1': boot_img.sha1,
2232 'recovery_size': recovery_img.size,
2233 'recovery_sha1': recovery_img.sha1,
2234 'boot_type': boot_type,
2235 'boot_device': boot_device,
2236 'recovery_type': recovery_type,
2237 'recovery_device': recovery_device,
2238 'bonus_args': bonus_args}
Doug Zongkerc9253822014-02-04 12:17:58 -08002239
2240 # The install script location moved from /system/etc to /system/bin
Tianjie Xu78de9f12017-06-20 16:52:54 -07002241 # in the L release.
2242 sh_location = "bin/install-recovery.sh"
Tao Bao9f0c8df2015-07-07 18:31:47 -07002243
Tao Bao32fcdab2018-10-12 10:30:39 -07002244 logger.info("putting script in %s", sh_location)
Doug Zongkerc9253822014-02-04 12:17:58 -08002245
2246 output_sink(sh_location, sh)
Yifan Hong10c530d2018-12-27 17:34:18 -08002247
2248
2249class DynamicPartitionUpdate(object):
2250 def __init__(self, src_group=None, tgt_group=None, progress=None,
2251 block_difference=None):
2252 self.src_group = src_group
2253 self.tgt_group = tgt_group
2254 self.progress = progress
2255 self.block_difference = block_difference
2256
2257 @property
2258 def src_size(self):
2259 if not self.block_difference:
2260 return 0
2261 return DynamicPartitionUpdate._GetSparseImageSize(self.block_difference.src)
2262
2263 @property
2264 def tgt_size(self):
2265 if not self.block_difference:
2266 return 0
2267 return DynamicPartitionUpdate._GetSparseImageSize(self.block_difference.tgt)
2268
2269 @staticmethod
2270 def _GetSparseImageSize(img):
2271 if not img:
2272 return 0
2273 return img.blocksize * img.total_blocks
2274
2275
2276class DynamicGroupUpdate(object):
2277 def __init__(self, src_size=None, tgt_size=None):
2278 # None: group does not exist. 0: no size limits.
2279 self.src_size = src_size
2280 self.tgt_size = tgt_size
2281
2282
2283class DynamicPartitionsDifference(object):
2284 def __init__(self, info_dict, block_diffs, progress_dict=None,
2285 source_info_dict=None):
2286 if progress_dict is None:
2287 progress_dict = dict()
2288
2289 self._remove_all_before_apply = False
2290 if source_info_dict is None:
2291 self._remove_all_before_apply = True
2292 source_info_dict = dict()
2293
2294 block_diff_dict = {e.partition:e for e in block_diffs}
2295 assert len(block_diff_dict) == len(block_diffs), \
2296 "Duplicated BlockDifference object for {}".format(
2297 [partition for partition, count in
2298 collections.Counter(e.partition for e in block_diffs).items()
2299 if count > 1])
2300
Yifan Hong79997e52019-01-23 16:56:19 -08002301 self._partition_updates = collections.OrderedDict()
Yifan Hong10c530d2018-12-27 17:34:18 -08002302
2303 for p, block_diff in block_diff_dict.items():
2304 self._partition_updates[p] = DynamicPartitionUpdate()
2305 self._partition_updates[p].block_difference = block_diff
2306
2307 for p, progress in progress_dict.items():
2308 if p in self._partition_updates:
2309 self._partition_updates[p].progress = progress
2310
2311 tgt_groups = shlex.split(info_dict.get(
2312 "super_partition_groups", "").strip())
2313 src_groups = shlex.split(source_info_dict.get(
2314 "super_partition_groups", "").strip())
2315
2316 for g in tgt_groups:
2317 for p in shlex.split(info_dict.get(
2318 "super_%s_partition_list" % g, "").strip()):
2319 assert p in self._partition_updates, \
2320 "{} is in target super_{}_partition_list but no BlockDifference " \
2321 "object is provided.".format(p, g)
2322 self._partition_updates[p].tgt_group = g
2323
2324 for g in src_groups:
2325 for p in shlex.split(source_info_dict.get(
2326 "super_%s_partition_list" % g, "").strip()):
2327 assert p in self._partition_updates, \
2328 "{} is in source super_{}_partition_list but no BlockDifference " \
2329 "object is provided.".format(p, g)
2330 self._partition_updates[p].src_group = g
2331
Yifan Hong45433e42019-01-18 13:55:25 -08002332 target_dynamic_partitions = set(shlex.split(info_dict.get(
2333 "dynamic_partition_list", "").strip()))
2334 block_diffs_with_target = set(p for p, u in self._partition_updates.items()
2335 if u.tgt_size)
2336 assert block_diffs_with_target == target_dynamic_partitions, \
2337 "Target Dynamic partitions: {}, BlockDifference with target: {}".format(
2338 list(target_dynamic_partitions), list(block_diffs_with_target))
2339
2340 source_dynamic_partitions = set(shlex.split(source_info_dict.get(
2341 "dynamic_partition_list", "").strip()))
2342 block_diffs_with_source = set(p for p, u in self._partition_updates.items()
2343 if u.src_size)
2344 assert block_diffs_with_source == source_dynamic_partitions, \
2345 "Source Dynamic partitions: {}, BlockDifference with source: {}".format(
2346 list(source_dynamic_partitions), list(block_diffs_with_source))
2347
Yifan Hong10c530d2018-12-27 17:34:18 -08002348 if self._partition_updates:
2349 logger.info("Updating dynamic partitions %s",
2350 self._partition_updates.keys())
2351
Yifan Hong79997e52019-01-23 16:56:19 -08002352 self._group_updates = collections.OrderedDict()
Yifan Hong10c530d2018-12-27 17:34:18 -08002353
2354 for g in tgt_groups:
2355 self._group_updates[g] = DynamicGroupUpdate()
2356 self._group_updates[g].tgt_size = int(info_dict.get(
2357 "super_%s_group_size" % g, "0").strip())
2358
2359 for g in src_groups:
2360 if g not in self._group_updates:
2361 self._group_updates[g] = DynamicGroupUpdate()
2362 self._group_updates[g].src_size = int(source_info_dict.get(
2363 "super_%s_group_size" % g, "0").strip())
2364
2365 self._Compute()
2366
2367 def WriteScript(self, script, output_zip, write_verify_script=False):
2368 script.Comment('--- Start patching dynamic partitions ---')
2369 for p, u in self._partition_updates.items():
2370 if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
2371 script.Comment('Patch partition %s' % p)
2372 u.block_difference.WriteScript(script, output_zip, progress=u.progress,
2373 write_verify_script=False)
2374
2375 op_list_path = MakeTempFile()
2376 with open(op_list_path, 'w') as f:
2377 for line in self._op_list:
2378 f.write('{}\n'.format(line))
2379
2380 ZipWrite(output_zip, op_list_path, "dynamic_partitions_op_list")
2381
2382 script.Comment('Update dynamic partition metadata')
2383 script.AppendExtra('assert(update_dynamic_partitions('
2384 'package_extract_file("dynamic_partitions_op_list")));')
2385
2386 if write_verify_script:
2387 for p, u in self._partition_updates.items():
2388 if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
2389 u.block_difference.WritePostInstallVerifyScript(script)
2390 script.AppendExtra('unmap_partition("%s");' % p) # ignore errors
2391
2392 for p, u in self._partition_updates.items():
2393 if u.tgt_size and u.src_size <= u.tgt_size:
2394 script.Comment('Patch partition %s' % p)
2395 u.block_difference.WriteScript(script, output_zip, progress=u.progress,
2396 write_verify_script=write_verify_script)
2397 if write_verify_script:
2398 script.AppendExtra('unmap_partition("%s");' % p) # ignore errors
2399
2400 script.Comment('--- End patching dynamic partitions ---')
2401
2402 def _Compute(self):
2403 self._op_list = list()
2404
2405 def append(line):
2406 self._op_list.append(line)
2407
2408 def comment(line):
2409 self._op_list.append("# %s" % line)
2410
2411 if self._remove_all_before_apply:
2412 comment('Remove all existing dynamic partitions and groups before '
2413 'applying full OTA')
2414 append('remove_all_groups')
2415
2416 for p, u in self._partition_updates.items():
2417 if u.src_group and not u.tgt_group:
2418 append('remove %s' % p)
2419
2420 for p, u in self._partition_updates.items():
2421 if u.src_group and u.tgt_group and u.src_group != u.tgt_group:
2422 comment('Move partition %s from %s to default' % (p, u.src_group))
2423 append('move %s default' % p)
2424
2425 for p, u in self._partition_updates.items():
2426 if u.src_size and u.tgt_size and u.src_size > u.tgt_size:
2427 comment('Shrink partition %s from %d to %d' %
2428 (p, u.src_size, u.tgt_size))
2429 append('resize %s %s' % (p, u.tgt_size))
2430
2431 for g, u in self._group_updates.items():
2432 if u.src_size is not None and u.tgt_size is None:
2433 append('remove_group %s' % g)
2434 if (u.src_size is not None and u.tgt_size is not None and
2435 u.src_size > u.tgt_size):
2436 comment('Shrink group %s from %d to %d' % (g, u.src_size, u.tgt_size))
2437 append('resize_group %s %d' % (g, u.tgt_size))
2438
2439 for g, u in self._group_updates.items():
2440 if u.src_size is None and u.tgt_size is not None:
2441 comment('Add group %s with maximum size %d' % (g, u.tgt_size))
2442 append('add_group %s %d' % (g, u.tgt_size))
2443 if (u.src_size is not None and u.tgt_size is not None and
2444 u.src_size < u.tgt_size):
2445 comment('Grow group %s from %d to %d' % (g, u.src_size, u.tgt_size))
2446 append('resize_group %s %d' % (g, u.tgt_size))
2447
2448 for p, u in self._partition_updates.items():
2449 if u.tgt_group and not u.src_group:
2450 comment('Add partition %s to group %s' % (p, u.tgt_group))
2451 append('add %s %s' % (p, u.tgt_group))
2452
2453 for p, u in self._partition_updates.items():
2454 if u.tgt_size and u.src_size < u.tgt_size:
2455 comment('Grow partition %s from %d to %d' % (p, u.src_size, u.tgt_size))
2456 append('resize %s %d' % (p, u.tgt_size))
2457
2458 for p, u in self._partition_updates.items():
2459 if u.src_group and u.tgt_group and u.src_group != u.tgt_group:
2460 comment('Move partition %s from default to %s' %
2461 (p, u.tgt_group))
2462 append('move %s %s' % (p, u.tgt_group))