blob: 844d376466cc321d8bdddf90af35e7ef8b6e28c3 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Doug Zongkerea5d7a92010-09-12 15:26:16 -070015import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070016import errno
Doug Zongkereef39442009-04-02 12:14:19 -070017import getopt
18import getpass
Doug Zongker05d3dea2009-06-22 11:32:31 -070019import imp
Doug Zongkereef39442009-04-02 12:14:19 -070020import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080021import platform
Doug Zongkereef39442009-04-02 12:14:19 -070022import re
T.R. Fullhart37e10522013-03-18 10:31:26 -070023import shlex
Doug Zongkereef39442009-04-02 12:14:19 -070024import shutil
25import subprocess
26import sys
27import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070028import threading
29import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070030import zipfile
Doug Zongkereef39442009-04-02 12:14:19 -070031
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070032import blockimgdiff
Dan Albert8b72aef2015-03-23 19:13:21 -070033import rangelib
Doug Zongkerab7ca1d2014-08-26 10:40:28 -070034
Tao Baof3282b42015-04-01 11:21:55 -070035from hashlib import sha1 as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080036
Doug Zongkereef39442009-04-02 12:14:19 -070037
Dan Albert8b72aef2015-03-23 19:13:21 -070038class Options(object):
39 def __init__(self):
40 platform_search_path = {
41 "linux2": "out/host/linux-x86",
42 "darwin": "out/host/darwin-x86",
Doug Zongker85448772014-09-09 14:59:20 -070043 }
Doug Zongker85448772014-09-09 14:59:20 -070044
Dan Albert8b72aef2015-03-23 19:13:21 -070045 self.search_path = platform_search_path.get(sys.platform, None)
46 self.signapk_path = "framework/signapk.jar" # Relative to search_path
47 self.extra_signapk_args = []
48 self.java_path = "java" # Use the one on the path by default.
49 self.java_args = "-Xmx2048m" # JVM Args
50 self.public_key_suffix = ".x509.pem"
51 self.private_key_suffix = ".pk8"
Dan Albertcd9ecc02015-03-27 16:37:23 -070052 # use otatools built boot_signer by default
53 self.boot_signer_path = "boot_signer"
Baligh Uddin601ddea2015-06-09 15:48:14 -070054 self.boot_signer_args = []
55 self.verity_signer_path = None
56 self.verity_signer_args = []
Dan Albert8b72aef2015-03-23 19:13:21 -070057 self.verbose = False
58 self.tempfiles = []
59 self.device_specific = None
60 self.extras = {}
61 self.info_dict = None
Tao Baoe09359a2015-10-13 16:37:12 -070062 self.source_info_dict = None
63 self.target_info_dict = None
Dan Albert8b72aef2015-03-23 19:13:21 -070064 self.worker_threads = None
Tao Bao116977c2015-08-07 19:49:45 -070065 # Stash size cannot exceed cache_size * threshold.
66 self.cache_size = None
67 self.stash_threshold = 0.8
Dan Albert8b72aef2015-03-23 19:13:21 -070068
69
70OPTIONS = Options()
Doug Zongkereef39442009-04-02 12:14:19 -070071
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080072
73# Values for "certificate" in apkcerts that mean special things.
74SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
75
76
Dan Albert8b72aef2015-03-23 19:13:21 -070077class ExternalError(RuntimeError):
78 pass
Doug Zongkereef39442009-04-02 12:14:19 -070079
80
81def Run(args, **kwargs):
82 """Create and return a subprocess.Popen object, printing the command
83 line on the terminal if -v was specified."""
84 if OPTIONS.verbose:
85 print " running: ", " ".join(args)
86 return subprocess.Popen(args, **kwargs)
87
88
Ying Wang7e6d4e42010-12-13 16:25:36 -080089def CloseInheritedPipes():
90 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
91 before doing other work."""
92 if platform.system() != "Darwin":
93 return
94 for d in range(3, 1025):
95 try:
96 stat = os.fstat(d)
97 if stat is not None:
98 pipebit = stat[0] & 0x1000
99 if pipebit != 0:
100 os.close(d)
101 except OSError:
102 pass
103
104
Tao Baoa6a3aa92015-07-09 11:51:16 -0700105def LoadInfoDict(input_file, input_dir=None):
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700106 """Read and parse the META/misc_info.txt key/value pairs from the
107 input target files and return a dict."""
108
Doug Zongkerc9253822014-02-04 12:17:58 -0800109 def read_helper(fn):
Dan Albert8b72aef2015-03-23 19:13:21 -0700110 if isinstance(input_file, zipfile.ZipFile):
111 return input_file.read(fn)
Doug Zongkerc9253822014-02-04 12:17:58 -0800112 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700113 path = os.path.join(input_file, *fn.split("/"))
Doug Zongkerc9253822014-02-04 12:17:58 -0800114 try:
115 with open(path) as f:
116 return f.read()
Dan Albert8b72aef2015-03-23 19:13:21 -0700117 except IOError as e:
Doug Zongkerc9253822014-02-04 12:17:58 -0800118 if e.errno == errno.ENOENT:
119 raise KeyError(fn)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700120 d = {}
121 try:
Michael Runge6e836112014-04-15 17:40:21 -0700122 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n"))
Doug Zongker37974732010-09-16 17:44:38 -0700123 except KeyError:
124 # ok if misc_info.txt doesn't exist
125 pass
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700126
Doug Zongker37974732010-09-16 17:44:38 -0700127 # backwards compatibility: These values used to be in their own
128 # files. Look for them, in case we're processing an old
129 # target_files zip.
130
131 if "mkyaffs2_extra_flags" not in d:
132 try:
Dan Albert8b72aef2015-03-23 19:13:21 -0700133 d["mkyaffs2_extra_flags"] = read_helper(
134 "META/mkyaffs2-extra-flags.txt").strip()
Doug Zongker37974732010-09-16 17:44:38 -0700135 except KeyError:
136 # ok if flags don't exist
137 pass
138
139 if "recovery_api_version" not in d:
140 try:
Dan Albert8b72aef2015-03-23 19:13:21 -0700141 d["recovery_api_version"] = read_helper(
142 "META/recovery-api-version.txt").strip()
Doug Zongker37974732010-09-16 17:44:38 -0700143 except KeyError:
144 raise ValueError("can't find recovery API version in input target-files")
145
146 if "tool_extensions" not in d:
147 try:
Doug Zongkerc9253822014-02-04 12:17:58 -0800148 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip()
Doug Zongker37974732010-09-16 17:44:38 -0700149 except KeyError:
150 # ok if extensions don't exist
151 pass
152
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800153 if "fstab_version" not in d:
154 d["fstab_version"] = "1"
155
Tao Baoa6a3aa92015-07-09 11:51:16 -0700156 # During building, we use the "file_contexts" in the out/ directory tree.
157 # It is no longer available when (re)generating from target_files zip. So
158 # when generating from target_files zip, we look for a copy under META/
159 # first, if not available search under BOOT/RAMDISK/. Note that we may need
160 # a different file_contexts to build images than the one running on device,
161 # such as when enabling system_root_image. In that case, we must have the
162 # one for building copied to META/.
163 if input_dir is not None:
164 fc_config = os.path.join(input_dir, "META", "file_contexts")
165 if not os.path.exists(fc_config):
166 fc_config = os.path.join(input_dir, "BOOT", "RAMDISK", "file_contexts")
167 if not os.path.exists(fc_config):
168 fc_config = None
169
170 if fc_config:
171 d["selinux_fc"] = fc_config
172
Doug Zongker37974732010-09-16 17:44:38 -0700173 try:
Doug Zongkerc9253822014-02-04 12:17:58 -0800174 data = read_helper("META/imagesizes.txt")
Doug Zongker37974732010-09-16 17:44:38 -0700175 for line in data.split("\n"):
Dan Albert8b72aef2015-03-23 19:13:21 -0700176 if not line:
177 continue
Doug Zongker1684d9c2010-09-17 07:44:38 -0700178 name, value = line.split(" ", 1)
Dan Albert8b72aef2015-03-23 19:13:21 -0700179 if not value:
180 continue
Doug Zongker37974732010-09-16 17:44:38 -0700181 if name == "blocksize":
182 d[name] = value
183 else:
184 d[name + "_size"] = value
185 except KeyError:
186 pass
187
188 def makeint(key):
189 if key in d:
190 d[key] = int(d[key], 0)
191
192 makeint("recovery_api_version")
193 makeint("blocksize")
194 makeint("system_size")
Daniel Rosenbergf4eabc32014-07-10 15:42:38 -0700195 makeint("vendor_size")
Doug Zongker37974732010-09-16 17:44:38 -0700196 makeint("userdata_size")
Ying Wang9f8e8db2011-11-04 11:37:01 -0700197 makeint("cache_size")
Doug Zongker37974732010-09-16 17:44:38 -0700198 makeint("recovery_size")
199 makeint("boot_size")
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800200 makeint("fstab_version")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700201
Tao Baob11d2c52015-07-21 18:01:20 -0700202 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"],
203 d.get("system_root_image", False))
Doug Zongkerc9253822014-02-04 12:17:58 -0800204 d["build.prop"] = LoadBuildProp(read_helper)
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700205 return d
206
Doug Zongkerc9253822014-02-04 12:17:58 -0800207def LoadBuildProp(read_helper):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700208 try:
Doug Zongkerc9253822014-02-04 12:17:58 -0800209 data = read_helper("SYSTEM/build.prop")
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700210 except KeyError:
211 print "Warning: could not find SYSTEM/build.prop in %s" % zip
212 data = ""
Michael Runge6e836112014-04-15 17:40:21 -0700213 return LoadDictionaryFromLines(data.split("\n"))
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700214
Michael Runge6e836112014-04-15 17:40:21 -0700215def LoadDictionaryFromLines(lines):
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700216 d = {}
Michael Runge6e836112014-04-15 17:40:21 -0700217 for line in lines:
Doug Zongker1eb74dd2012-08-16 16:19:00 -0700218 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700219 if not line or line.startswith("#"):
220 continue
Ying Wang114b46f2014-04-15 11:24:00 -0700221 if "=" in line:
222 name, value = line.split("=", 1)
223 d[name] = value
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700224 return d
225
Daniel Rosenbergb3b8ce62015-06-05 17:59:27 -0700226def LoadRecoveryFSTab(read_helper, fstab_version, system_root_image=False):
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700227 class Partition(object):
Tao Bao548eb762015-06-10 12:32:41 -0700228 def __init__(self, mount_point, fs_type, device, length, device2, context):
Dan Albert8b72aef2015-03-23 19:13:21 -0700229 self.mount_point = mount_point
230 self.fs_type = fs_type
231 self.device = device
232 self.length = length
233 self.device2 = device2
Tao Bao548eb762015-06-10 12:32:41 -0700234 self.context = context
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700235
236 try:
Doug Zongkerc9253822014-02-04 12:17:58 -0800237 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab")
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700238 except KeyError:
Doug Zongkerc9253822014-02-04 12:17:58 -0800239 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab"
Jeff Davidson033fbe22011-10-26 18:08:09 -0700240 data = ""
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700241
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800242 if fstab_version == 1:
243 d = {}
244 for line in data.split("\n"):
245 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700246 if not line or line.startswith("#"):
247 continue
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800248 pieces = line.split()
Dan Albert8b72aef2015-03-23 19:13:21 -0700249 if not 3 <= len(pieces) <= 4:
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800250 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800251 options = None
252 if len(pieces) >= 4:
253 if pieces[3].startswith("/"):
Dan Albert8b72aef2015-03-23 19:13:21 -0700254 device2 = pieces[3]
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800255 if len(pieces) >= 5:
256 options = pieces[4]
257 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700258 device2 = None
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800259 options = pieces[3]
Doug Zongker086cbb02011-02-17 15:54:20 -0800260 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700261 device2 = None
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700262
Dan Albert8b72aef2015-03-23 19:13:21 -0700263 mount_point = pieces[0]
264 length = 0
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800265 if options:
266 options = options.split(",")
267 for i in options:
268 if i.startswith("length="):
Dan Albert8b72aef2015-03-23 19:13:21 -0700269 length = int(i[7:])
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800270 else:
Dan Albert8b72aef2015-03-23 19:13:21 -0700271 print "%s: unknown option \"%s\"" % (mount_point, i)
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800272
Dan Albert8b72aef2015-03-23 19:13:21 -0700273 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[1],
274 device=pieces[2], length=length,
275 device2=device2)
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800276
277 elif fstab_version == 2:
278 d = {}
279 for line in data.split("\n"):
280 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700281 if not line or line.startswith("#"):
282 continue
Tao Bao548eb762015-06-10 12:32:41 -0700283 # <src> <mnt_point> <type> <mnt_flags and options> <fs_mgr_flags>
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800284 pieces = line.split()
285 if len(pieces) != 5:
286 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
287
288 # Ignore entries that are managed by vold
289 options = pieces[4]
Dan Albert8b72aef2015-03-23 19:13:21 -0700290 if "voldmanaged=" in options:
291 continue
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800292
293 # It's a good line, parse it
Dan Albert8b72aef2015-03-23 19:13:21 -0700294 length = 0
Doug Zongker086cbb02011-02-17 15:54:20 -0800295 options = options.split(",")
296 for i in options:
297 if i.startswith("length="):
Dan Albert8b72aef2015-03-23 19:13:21 -0700298 length = int(i[7:])
Doug Zongker086cbb02011-02-17 15:54:20 -0800299 else:
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800300 # Ignore all unknown options in the unified fstab
301 continue
Doug Zongker086cbb02011-02-17 15:54:20 -0800302
Tao Bao548eb762015-06-10 12:32:41 -0700303 mount_flags = pieces[3]
304 # Honor the SELinux context if present.
305 context = None
306 for i in mount_flags.split(","):
307 if i.startswith("context="):
308 context = i
309
Dan Albert8b72aef2015-03-23 19:13:21 -0700310 mount_point = pieces[1]
311 d[mount_point] = Partition(mount_point=mount_point, fs_type=pieces[2],
Tao Bao548eb762015-06-10 12:32:41 -0700312 device=pieces[0], length=length,
313 device2=None, context=context)
Ken Sumrall3b07cf12013-02-19 17:35:29 -0800314
315 else:
316 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,))
317
Daniel Rosenbergb3b8ce62015-06-05 17:59:27 -0700318 # / is used for the system mount point when the root directory is included in
Tao Baob11d2c52015-07-21 18:01:20 -0700319 # system. Other areas assume system is always at "/system" so point /system
320 # at /.
Daniel Rosenbergb3b8ce62015-06-05 17:59:27 -0700321 if system_root_image:
322 assert not d.has_key("/system") and d.has_key("/")
323 d["/system"] = d["/"]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700324 return d
325
326
Doug Zongker37974732010-09-16 17:44:38 -0700327def DumpInfoDict(d):
328 for k, v in sorted(d.items()):
329 print "%-25s = (%s) %s" % (k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700330
Dan Albert8b72aef2015-03-23 19:13:21 -0700331
Tao Baob11d2c52015-07-21 18:01:20 -0700332def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
333 has_ramdisk=False):
334 """Build a bootable image from the specified sourcedir.
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700335
Tao Baob11d2c52015-07-21 18:01:20 -0700336 Take a kernel, cmdline, and optionally a ramdisk directory from the input (in
337 'sourcedir'), and turn them into a boot image. Return the image data, or
338 None if sourcedir does not appear to contains files for building the
339 requested image."""
340
341 def make_ramdisk():
342 ramdisk_img = tempfile.NamedTemporaryFile()
343
344 if os.access(fs_config_file, os.F_OK):
345 cmd = ["mkbootfs", "-f", fs_config_file,
346 os.path.join(sourcedir, "RAMDISK")]
347 else:
348 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")]
349 p1 = Run(cmd, stdout=subprocess.PIPE)
350 p2 = Run(["minigzip"], stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
351
352 p2.wait()
353 p1.wait()
354 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (sourcedir,)
355 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (sourcedir,)
356
357 return ramdisk_img
358
359 if not os.access(os.path.join(sourcedir, "kernel"), os.F_OK):
360 return None
361
362 if has_ramdisk and not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK):
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700363 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700364
Doug Zongkerd5131602012-08-02 14:46:42 -0700365 if info_dict is None:
366 info_dict = OPTIONS.info_dict
367
Doug Zongkereef39442009-04-02 12:14:19 -0700368 img = tempfile.NamedTemporaryFile()
369
Tao Baob11d2c52015-07-21 18:01:20 -0700370 if has_ramdisk:
371 ramdisk_img = make_ramdisk()
Doug Zongkereef39442009-04-02 12:14:19 -0700372
Bjorn Andersson612e2cd2012-11-25 16:53:44 -0800373 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set
374 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg"
375
376 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")]
Doug Zongker38a649f2009-06-17 09:07:09 -0700377
Benoit Fradina45a8682014-07-14 21:00:43 +0200378 fn = os.path.join(sourcedir, "second")
379 if os.access(fn, os.F_OK):
380 cmd.append("--second")
381 cmd.append(fn)
382
Doug Zongker171f1cd2009-06-15 22:36:37 -0700383 fn = os.path.join(sourcedir, "cmdline")
384 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700385 cmd.append("--cmdline")
386 cmd.append(open(fn).read().rstrip("\n"))
387
388 fn = os.path.join(sourcedir, "base")
389 if os.access(fn, os.F_OK):
390 cmd.append("--base")
391 cmd.append(open(fn).read().rstrip("\n"))
392
Ying Wang4de6b5b2010-08-25 14:29:34 -0700393 fn = os.path.join(sourcedir, "pagesize")
394 if os.access(fn, os.F_OK):
395 cmd.append("--pagesize")
396 cmd.append(open(fn).read().rstrip("\n"))
397
Doug Zongkerd5131602012-08-02 14:46:42 -0700398 args = info_dict.get("mkbootimg_args", None)
399 if args and args.strip():
Jianxun Zhang09849492013-04-17 15:19:19 -0700400 cmd.extend(shlex.split(args))
Doug Zongkerd5131602012-08-02 14:46:42 -0700401
Tao Baob11d2c52015-07-21 18:01:20 -0700402 if has_ramdisk:
403 cmd.extend(["--ramdisk", ramdisk_img.name])
404
Tao Baod95e9fd2015-03-29 23:07:41 -0700405 img_unsigned = None
406 if info_dict.get("vboot", None):
407 img_unsigned = tempfile.NamedTemporaryFile()
Tao Baob11d2c52015-07-21 18:01:20 -0700408 cmd.extend(["--output", img_unsigned.name])
Tao Baod95e9fd2015-03-29 23:07:41 -0700409 else:
Tao Baob11d2c52015-07-21 18:01:20 -0700410 cmd.extend(["--output", img.name])
Doug Zongker38a649f2009-06-17 09:07:09 -0700411
412 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700413 p.communicate()
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700414 assert p.returncode == 0, "mkbootimg of %s image failed" % (
415 os.path.basename(sourcedir),)
Doug Zongkereef39442009-04-02 12:14:19 -0700416
Sami Tolvanen8b3f08b2015-04-07 15:08:59 +0100417 if (info_dict.get("boot_signer", None) == "true" and
418 info_dict.get("verity_key", None)):
Geremy Condra95ebe7a2014-08-19 17:27:56 -0700419 path = "/" + os.path.basename(sourcedir).lower()
Baligh Uddin601ddea2015-06-09 15:48:14 -0700420 cmd = [OPTIONS.boot_signer_path]
421 cmd.extend(OPTIONS.boot_signer_args)
422 cmd.extend([path, img.name,
423 info_dict["verity_key"] + ".pk8",
424 info_dict["verity_key"] + ".x509.pem", img.name])
Geremy Condra95ebe7a2014-08-19 17:27:56 -0700425 p = Run(cmd, stdout=subprocess.PIPE)
426 p.communicate()
427 assert p.returncode == 0, "boot_signer of %s image failed" % path
428
Tao Baod95e9fd2015-03-29 23:07:41 -0700429 # Sign the image if vboot is non-empty.
430 elif info_dict.get("vboot", None):
431 path = "/" + os.path.basename(sourcedir).lower()
432 img_keyblock = tempfile.NamedTemporaryFile()
433 cmd = [info_dict["vboot_signer_cmd"], info_dict["futility"],
434 img_unsigned.name, info_dict["vboot_key"] + ".vbpubk",
Furquan Shaikh852b8de2015-08-10 11:43:45 -0700435 info_dict["vboot_key"] + ".vbprivk",
436 info_dict["vboot_subkey"] + ".vbprivk",
437 img_keyblock.name,
Tao Baod95e9fd2015-03-29 23:07:41 -0700438 img.name]
439 p = Run(cmd, stdout=subprocess.PIPE)
440 p.communicate()
441 assert p.returncode == 0, "vboot_signer of %s image failed" % path
442
Tao Baof3282b42015-04-01 11:21:55 -0700443 # Clean up the temp files.
444 img_unsigned.close()
445 img_keyblock.close()
446
Doug Zongkereef39442009-04-02 12:14:19 -0700447 img.seek(os.SEEK_SET, 0)
448 data = img.read()
449
Tao Baob11d2c52015-07-21 18:01:20 -0700450 if has_ramdisk:
451 ramdisk_img.close()
Doug Zongkereef39442009-04-02 12:14:19 -0700452 img.close()
453
454 return data
455
456
Doug Zongkerd5131602012-08-02 14:46:42 -0700457def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,
458 info_dict=None):
Tao Baob11d2c52015-07-21 18:01:20 -0700459 """Return a File object with the desired bootable image.
460
461 Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 'prebuilt_name',
462 otherwise look for it under 'unpack_dir'/IMAGES, otherwise construct it from
463 the source files in 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700464
Doug Zongker55d93282011-01-25 17:03:34 -0800465 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
466 if os.path.exists(prebuilt_path):
Doug Zongker6f1d0312014-08-22 08:07:12 -0700467 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,)
Doug Zongker55d93282011-01-25 17:03:34 -0800468 return File.FromLocalFile(name, prebuilt_path)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700469
470 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name)
471 if os.path.exists(prebuilt_path):
472 print "using prebuilt %s from IMAGES..." % (prebuilt_name,)
473 return File.FromLocalFile(name, prebuilt_path)
474
475 print "building image from target_files %s..." % (tree_subdir,)
Tao Baob11d2c52015-07-21 18:01:20 -0700476
477 if info_dict is None:
478 info_dict = OPTIONS.info_dict
479
480 # With system_root_image == "true", we don't pack ramdisk into the boot image.
481 has_ramdisk = (info_dict.get("system_root_image", None) != "true" or
482 prebuilt_name != "boot.img")
483
Doug Zongker6f1d0312014-08-22 08:07:12 -0700484 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt"
Tao Baob11d2c52015-07-21 18:01:20 -0700485 data = _BuildBootableImage(os.path.join(unpack_dir, tree_subdir),
486 os.path.join(unpack_dir, fs_config),
487 info_dict, has_ramdisk)
Doug Zongker6f1d0312014-08-22 08:07:12 -0700488 if data:
489 return File(name, data)
490 return None
Doug Zongker55d93282011-01-25 17:03:34 -0800491
Doug Zongkereef39442009-04-02 12:14:19 -0700492
Doug Zongker75f17362009-12-08 13:46:44 -0800493def UnzipTemp(filename, pattern=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800494 """Unzip the given archive into a temporary directory and return the name.
495
496 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
497 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
498
499 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
500 main file), open for reading.
501 """
Doug Zongkereef39442009-04-02 12:14:19 -0700502
503 tmp = tempfile.mkdtemp(prefix="targetfiles-")
504 OPTIONS.tempfiles.append(tmp)
Doug Zongker55d93282011-01-25 17:03:34 -0800505
506 def unzip_to_dir(filename, dirname):
507 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
508 if pattern is not None:
509 cmd.append(pattern)
510 p = Run(cmd, stdout=subprocess.PIPE)
511 p.communicate()
512 if p.returncode != 0:
513 raise ExternalError("failed to unzip input target-files \"%s\"" %
514 (filename,))
515
516 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
517 if m:
518 unzip_to_dir(m.group(1), tmp)
519 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
520 filename = m.group(1)
521 else:
522 unzip_to_dir(filename, tmp)
523
524 return tmp, zipfile.ZipFile(filename, "r")
Doug Zongkereef39442009-04-02 12:14:19 -0700525
526
527def GetKeyPasswords(keylist):
528 """Given a list of keys, prompt the user to enter passwords for
529 those which require them. Return a {key: password} dict. password
530 will be None if the key has no password."""
531
Doug Zongker8ce7c252009-05-22 13:34:54 -0700532 no_passwords = []
533 need_passwords = []
T.R. Fullhart37e10522013-03-18 10:31:26 -0700534 key_passwords = {}
Doug Zongkereef39442009-04-02 12:14:19 -0700535 devnull = open("/dev/null", "w+b")
536 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800537 # We don't need a password for things that aren't really keys.
538 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700539 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700540 continue
541
T.R. Fullhart37e10522013-03-18 10:31:26 -0700542 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
Doug Zongker602a84e2009-06-18 08:35:12 -0700543 "-inform", "DER", "-nocrypt"],
544 stdin=devnull.fileno(),
545 stdout=devnull.fileno(),
546 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700547 p.communicate()
548 if p.returncode == 0:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700549 # Definitely an unencrypted key.
Doug Zongker8ce7c252009-05-22 13:34:54 -0700550 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700551 else:
T.R. Fullhart37e10522013-03-18 10:31:26 -0700552 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix,
553 "-inform", "DER", "-passin", "pass:"],
554 stdin=devnull.fileno(),
555 stdout=devnull.fileno(),
556 stderr=subprocess.PIPE)
Dan Albert8b72aef2015-03-23 19:13:21 -0700557 _, stderr = p.communicate()
T.R. Fullhart37e10522013-03-18 10:31:26 -0700558 if p.returncode == 0:
559 # Encrypted key with empty string as password.
560 key_passwords[k] = ''
561 elif stderr.startswith('Error decrypting key'):
562 # Definitely encrypted key.
563 # It would have said "Error reading key" if it didn't parse correctly.
564 need_passwords.append(k)
565 else:
566 # Potentially, a type of key that openssl doesn't understand.
567 # We'll let the routines in signapk.jar handle it.
568 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700569 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700570
T.R. Fullhart37e10522013-03-18 10:31:26 -0700571 key_passwords.update(PasswordManager().GetPasswords(need_passwords))
Doug Zongker8ce7c252009-05-22 13:34:54 -0700572 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700573 return key_passwords
574
575
Doug Zongker951495f2009-08-14 12:44:19 -0700576def SignFile(input_name, output_name, key, password, align=None,
577 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700578 """Sign the input_name zip/jar/apk, producing output_name. Use the
579 given key and password (the latter may be None if the key does not
580 have a password.
581
582 If align is an integer > 1, zipalign is run to align stored files in
583 the output zip on 'align'-byte boundaries.
Doug Zongker951495f2009-08-14 12:44:19 -0700584
585 If whole_file is true, use the "-w" option to SignApk to embed a
586 signature that covers the whole file in the archive comment of the
587 zip file.
Doug Zongkereef39442009-04-02 12:14:19 -0700588 """
Doug Zongker951495f2009-08-14 12:44:19 -0700589
Doug Zongkereef39442009-04-02 12:14:19 -0700590 if align == 0 or align == 1:
591 align = None
592
593 if align:
594 temp = tempfile.NamedTemporaryFile()
595 sign_name = temp.name
596 else:
597 sign_name = output_name
598
Baligh Uddin339ee492014-09-05 11:18:07 -0700599 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar",
T.R. Fullhart37e10522013-03-18 10:31:26 -0700600 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)]
601 cmd.extend(OPTIONS.extra_signapk_args)
Doug Zongker951495f2009-08-14 12:44:19 -0700602 if whole_file:
603 cmd.append("-w")
T.R. Fullhart37e10522013-03-18 10:31:26 -0700604 cmd.extend([key + OPTIONS.public_key_suffix,
605 key + OPTIONS.private_key_suffix,
Doug Zongker951495f2009-08-14 12:44:19 -0700606 input_name, sign_name])
607
608 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700609 if password is not None:
610 password += "\n"
611 p.communicate(password)
612 if p.returncode != 0:
613 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
614
615 if align:
Brian Carlstrom903186f2015-05-22 15:51:19 -0700616 p = Run(["zipalign", "-f", "-p", str(align), sign_name, output_name])
Doug Zongkereef39442009-04-02 12:14:19 -0700617 p.communicate()
618 if p.returncode != 0:
619 raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
620 temp.close()
621
622
Doug Zongker37974732010-09-16 17:44:38 -0700623def CheckSize(data, target, info_dict):
Doug Zongkereef39442009-04-02 12:14:19 -0700624 """Check the data string passed against the max size limit, if
625 any, for the given target. Raise exception if the data is too big.
626 Print a warning if the data is nearing the maximum size."""
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700627
Dan Albert8b72aef2015-03-23 19:13:21 -0700628 if target.endswith(".img"):
629 target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700630 mount_point = "/" + target
631
Ying Wangf8824af2014-06-03 14:07:27 -0700632 fs_type = None
633 limit = None
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700634 if info_dict["fstab"]:
Dan Albert8b72aef2015-03-23 19:13:21 -0700635 if mount_point == "/userdata":
636 mount_point = "/data"
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700637 p = info_dict["fstab"][mount_point]
638 fs_type = p.fs_type
Andrew Boie0f9aec82012-02-14 09:32:52 -0800639 device = p.device
640 if "/" in device:
641 device = device[device.rfind("/")+1:]
642 limit = info_dict.get(device + "_size", None)
Dan Albert8b72aef2015-03-23 19:13:21 -0700643 if not fs_type or not limit:
644 return
Doug Zongkereef39442009-04-02 12:14:19 -0700645
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700646 if fs_type == "yaffs2":
647 # image size should be increased by 1/64th to account for the
648 # spare area (64 bytes per 2k page)
649 limit = limit / 2048 * (2048+64)
Andrew Boie0f9aec82012-02-14 09:32:52 -0800650 size = len(data)
651 pct = float(size) * 100.0 / limit
652 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
653 if pct >= 99.0:
654 raise ExternalError(msg)
655 elif pct >= 95.0:
656 print
657 print " WARNING: ", msg
658 print
659 elif OPTIONS.verbose:
660 print " ", msg
Doug Zongkereef39442009-04-02 12:14:19 -0700661
662
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800663def ReadApkCerts(tf_zip):
664 """Given a target_files ZipFile, parse the META/apkcerts.txt file
665 and return a {package: cert} dict."""
666 certmap = {}
667 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
668 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700669 if not line:
670 continue
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800671 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
672 r'private_key="(.*)"$', line)
673 if m:
674 name, cert, privkey = m.groups()
T.R. Fullhart37e10522013-03-18 10:31:26 -0700675 public_key_suffix_len = len(OPTIONS.public_key_suffix)
676 private_key_suffix_len = len(OPTIONS.private_key_suffix)
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800677 if cert in SPECIAL_CERT_STRINGS and not privkey:
678 certmap[name] = cert
T.R. Fullhart37e10522013-03-18 10:31:26 -0700679 elif (cert.endswith(OPTIONS.public_key_suffix) and
680 privkey.endswith(OPTIONS.private_key_suffix) and
681 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]):
682 certmap[name] = cert[:-public_key_suffix_len]
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800683 else:
684 raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
685 return certmap
686
687
Doug Zongkereef39442009-04-02 12:14:19 -0700688COMMON_DOCSTRING = """
689 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700690 Prepend <dir>/bin to the list of places to search for binaries
691 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700692
Doug Zongker05d3dea2009-06-22 11:32:31 -0700693 -s (--device_specific) <file>
694 Path to the python module containing device-specific
695 releasetools code.
696
Doug Zongker8bec09e2009-11-30 15:37:14 -0800697 -x (--extra) <key=value>
698 Add a key/value pair to the 'extras' dict, which device-specific
699 extension code may look at.
700
Doug Zongkereef39442009-04-02 12:14:19 -0700701 -v (--verbose)
702 Show command lines being executed.
703
704 -h (--help)
705 Display this usage message and exit.
706"""
707
708def Usage(docstring):
709 print docstring.rstrip("\n")
710 print COMMON_DOCSTRING
711
712
713def ParseOptions(argv,
714 docstring,
715 extra_opts="", extra_long_opts=(),
716 extra_option_handler=None):
717 """Parse the options in argv and return any arguments that aren't
718 flags. docstring is the calling module's docstring, to be displayed
719 for errors and -h. extra_opts and extra_long_opts are for flags
720 defined by the caller, which are processed by passing them to
721 extra_option_handler."""
722
723 try:
724 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800725 argv, "hvp:s:x:" + extra_opts,
T.R. Fullhart37e10522013-03-18 10:31:26 -0700726 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=",
Baligh Uddinbdc2e312014-09-05 17:36:20 -0700727 "java_path=", "java_args=", "public_key_suffix=",
Baligh Uddin601ddea2015-06-09 15:48:14 -0700728 "private_key_suffix=", "boot_signer_path=", "boot_signer_args=",
729 "verity_signer_path=", "verity_signer_args=", "device_specific=",
Baligh Uddine2048682014-11-20 09:52:05 -0800730 "extra="] +
T.R. Fullhart37e10522013-03-18 10:31:26 -0700731 list(extra_long_opts))
Dan Albert8b72aef2015-03-23 19:13:21 -0700732 except getopt.GetoptError as err:
Doug Zongkereef39442009-04-02 12:14:19 -0700733 Usage(docstring)
734 print "**", str(err), "**"
735 sys.exit(2)
736
Doug Zongkereef39442009-04-02 12:14:19 -0700737 for o, a in opts:
738 if o in ("-h", "--help"):
739 Usage(docstring)
740 sys.exit()
741 elif o in ("-v", "--verbose"):
742 OPTIONS.verbose = True
743 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -0700744 OPTIONS.search_path = a
T.R. Fullhart37e10522013-03-18 10:31:26 -0700745 elif o in ("--signapk_path",):
746 OPTIONS.signapk_path = a
747 elif o in ("--extra_signapk_args",):
748 OPTIONS.extra_signapk_args = shlex.split(a)
749 elif o in ("--java_path",):
750 OPTIONS.java_path = a
Baligh Uddin339ee492014-09-05 11:18:07 -0700751 elif o in ("--java_args",):
752 OPTIONS.java_args = a
T.R. Fullhart37e10522013-03-18 10:31:26 -0700753 elif o in ("--public_key_suffix",):
754 OPTIONS.public_key_suffix = a
755 elif o in ("--private_key_suffix",):
756 OPTIONS.private_key_suffix = a
Baligh Uddine2048682014-11-20 09:52:05 -0800757 elif o in ("--boot_signer_path",):
758 OPTIONS.boot_signer_path = a
Baligh Uddin601ddea2015-06-09 15:48:14 -0700759 elif o in ("--boot_signer_args",):
760 OPTIONS.boot_signer_args = shlex.split(a)
761 elif o in ("--verity_signer_path",):
762 OPTIONS.verity_signer_path = a
763 elif o in ("--verity_signer_args",):
764 OPTIONS.verity_signer_args = shlex.split(a)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700765 elif o in ("-s", "--device_specific"):
766 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -0800767 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -0800768 key, value = a.split("=", 1)
769 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -0700770 else:
771 if extra_option_handler is None or not extra_option_handler(o, a):
772 assert False, "unknown option \"%s\"" % (o,)
773
Doug Zongker85448772014-09-09 14:59:20 -0700774 if OPTIONS.search_path:
775 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
776 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -0700777
778 return args
779
780
Doug Zongkerfc44a512014-08-26 13:10:25 -0700781def MakeTempFile(prefix=None, suffix=None):
782 """Make a temp file and add it to the list of things to be deleted
783 when Cleanup() is called. Return the filename."""
784 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix)
785 os.close(fd)
786 OPTIONS.tempfiles.append(fn)
787 return fn
788
789
Doug Zongkereef39442009-04-02 12:14:19 -0700790def Cleanup():
791 for i in OPTIONS.tempfiles:
792 if os.path.isdir(i):
793 shutil.rmtree(i)
794 else:
795 os.remove(i)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700796
797
798class PasswordManager(object):
799 def __init__(self):
800 self.editor = os.getenv("EDITOR", None)
801 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
802
803 def GetPasswords(self, items):
804 """Get passwords corresponding to each string in 'items',
805 returning a dict. (The dict may have keys in addition to the
806 values in 'items'.)
807
808 Uses the passwords in $ANDROID_PW_FILE if available, letting the
809 user edit that file to add more needed passwords. If no editor is
810 available, or $ANDROID_PW_FILE isn't define, prompts the user
811 interactively in the ordinary way.
812 """
813
814 current = self.ReadFile()
815
816 first = True
817 while True:
818 missing = []
819 for i in items:
820 if i not in current or not current[i]:
821 missing.append(i)
822 # Are all the passwords already in the file?
Dan Albert8b72aef2015-03-23 19:13:21 -0700823 if not missing:
824 return current
Doug Zongker8ce7c252009-05-22 13:34:54 -0700825
826 for i in missing:
827 current[i] = ""
828
829 if not first:
830 print "key file %s still missing some passwords." % (self.pwfile,)
831 answer = raw_input("try to edit again? [y]> ").strip()
832 if answer and answer[0] not in 'yY':
833 raise RuntimeError("key passwords unavailable")
834 first = False
835
836 current = self.UpdateAndReadFile(current)
837
Dan Albert8b72aef2015-03-23 19:13:21 -0700838 def PromptResult(self, current): # pylint: disable=no-self-use
Doug Zongker8ce7c252009-05-22 13:34:54 -0700839 """Prompt the user to enter a value (password) for each key in
840 'current' whose value is fales. Returns a new dict with all the
841 values.
842 """
843 result = {}
844 for k, v in sorted(current.iteritems()):
845 if v:
846 result[k] = v
847 else:
848 while True:
Dan Albert8b72aef2015-03-23 19:13:21 -0700849 result[k] = getpass.getpass(
850 "Enter password for %s key> " % k).strip()
851 if result[k]:
852 break
Doug Zongker8ce7c252009-05-22 13:34:54 -0700853 return result
854
855 def UpdateAndReadFile(self, current):
856 if not self.editor or not self.pwfile:
857 return self.PromptResult(current)
858
859 f = open(self.pwfile, "w")
Dan Albert8b72aef2015-03-23 19:13:21 -0700860 os.chmod(self.pwfile, 0o600)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700861 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
862 f.write("# (Additional spaces are harmless.)\n\n")
863
864 first_line = None
Dan Albert8b72aef2015-03-23 19:13:21 -0700865 sorted_list = sorted([(not v, k, v) for (k, v) in current.iteritems()])
866 for i, (_, k, v) in enumerate(sorted_list):
Doug Zongker8ce7c252009-05-22 13:34:54 -0700867 f.write("[[[ %s ]]] %s\n" % (v, k))
868 if not v and first_line is None:
869 # position cursor on first line with no password.
870 first_line = i + 4
871 f.close()
872
873 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
874 _, _ = p.communicate()
875
876 return self.ReadFile()
877
878 def ReadFile(self):
879 result = {}
Dan Albert8b72aef2015-03-23 19:13:21 -0700880 if self.pwfile is None:
881 return result
Doug Zongker8ce7c252009-05-22 13:34:54 -0700882 try:
883 f = open(self.pwfile, "r")
884 for line in f:
885 line = line.strip()
Dan Albert8b72aef2015-03-23 19:13:21 -0700886 if not line or line[0] == '#':
887 continue
Doug Zongker8ce7c252009-05-22 13:34:54 -0700888 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
889 if not m:
890 print "failed to parse password file: ", line
891 else:
892 result[m.group(2)] = m.group(1)
893 f.close()
Dan Albert8b72aef2015-03-23 19:13:21 -0700894 except IOError as e:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700895 if e.errno != errno.ENOENT:
896 print "error reading password file: ", str(e)
897 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -0700898
899
Dan Albert8e0178d2015-01-27 15:53:15 -0800900def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
901 compress_type=None):
902 import datetime
903
904 # http://b/18015246
905 # Python 2.7's zipfile implementation wrongly thinks that zip64 is required
906 # for files larger than 2GiB. We can work around this by adjusting their
907 # limit. Note that `zipfile.writestr()` will not work for strings larger than
908 # 2GiB. The Python interpreter sometimes rejects strings that large (though
909 # it isn't clear to me exactly what circumstances cause this).
910 # `zipfile.write()` must be used directly to work around this.
911 #
912 # This mess can be avoided if we port to python3.
913 saved_zip64_limit = zipfile.ZIP64_LIMIT
914 zipfile.ZIP64_LIMIT = (1 << 32) - 1
915
916 if compress_type is None:
917 compress_type = zip_file.compression
918 if arcname is None:
919 arcname = filename
920
921 saved_stat = os.stat(filename)
922
923 try:
924 # `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the
925 # file to be zipped and reset it when we're done.
926 os.chmod(filename, perms)
927
928 # Use a fixed timestamp so the output is repeatable.
929 epoch = datetime.datetime.fromtimestamp(0)
930 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
931 os.utime(filename, (timestamp, timestamp))
932
933 zip_file.write(filename, arcname=arcname, compress_type=compress_type)
934 finally:
935 os.chmod(filename, saved_stat.st_mode)
936 os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime))
937 zipfile.ZIP64_LIMIT = saved_zip64_limit
938
939
Tao Bao58c1b962015-05-20 09:32:18 -0700940def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
Tao Baof3282b42015-04-01 11:21:55 -0700941 compress_type=None):
942 """Wrap zipfile.writestr() function to work around the zip64 limit.
943
944 Even with the ZIP64_LIMIT workaround, it won't allow writing a string
945 longer than 2GiB. It gives 'OverflowError: size does not fit in an int'
946 when calling crc32(bytes).
947
948 But it still works fine to write a shorter string into a large zip file.
949 We should use ZipWrite() whenever possible, and only use ZipWriteStr()
950 when we know the string won't be too long.
951 """
952
953 saved_zip64_limit = zipfile.ZIP64_LIMIT
954 zipfile.ZIP64_LIMIT = (1 << 32) - 1
955
956 if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):
957 zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)
Dan Albert8b72aef2015-03-23 19:13:21 -0700958 zinfo.compress_type = zip_file.compression
Tao Bao58c1b962015-05-20 09:32:18 -0700959 if perms is None:
960 perms = 0o644
Geremy Condra36bd3652014-02-06 19:45:10 -0800961 else:
Tao Baof3282b42015-04-01 11:21:55 -0700962 zinfo = zinfo_or_arcname
963
964 # If compress_type is given, it overrides the value in zinfo.
965 if compress_type is not None:
966 zinfo.compress_type = compress_type
967
Tao Bao58c1b962015-05-20 09:32:18 -0700968 # If perms is given, it has a priority.
969 if perms is not None:
970 zinfo.external_attr = perms << 16
971
Tao Baof3282b42015-04-01 11:21:55 -0700972 # Use a fixed timestamp so the output is repeatable.
Tao Baof3282b42015-04-01 11:21:55 -0700973 zinfo.date_time = (2009, 1, 1, 0, 0, 0)
974
Dan Albert8b72aef2015-03-23 19:13:21 -0700975 zip_file.writestr(zinfo, data)
Tao Baof3282b42015-04-01 11:21:55 -0700976 zipfile.ZIP64_LIMIT = saved_zip64_limit
977
978
979def ZipClose(zip_file):
980 # http://b/18015246
981 # zipfile also refers to ZIP64_LIMIT during close() when it writes out the
982 # central directory.
983 saved_zip64_limit = zipfile.ZIP64_LIMIT
984 zipfile.ZIP64_LIMIT = (1 << 32) - 1
985
986 zip_file.close()
987
988 zipfile.ZIP64_LIMIT = saved_zip64_limit
Doug Zongker05d3dea2009-06-22 11:32:31 -0700989
990
991class DeviceSpecificParams(object):
992 module = None
993 def __init__(self, **kwargs):
994 """Keyword arguments to the constructor become attributes of this
995 object, which is passed to all functions in the device-specific
996 module."""
997 for k, v in kwargs.iteritems():
998 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -0800999 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -07001000
1001 if self.module is None:
1002 path = OPTIONS.device_specific
Dan Albert8b72aef2015-03-23 19:13:21 -07001003 if not path:
1004 return
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001005 try:
1006 if os.path.isdir(path):
1007 info = imp.find_module("releasetools", [path])
1008 else:
1009 d, f = os.path.split(path)
1010 b, x = os.path.splitext(f)
1011 if x == ".py":
1012 f = b
1013 info = imp.find_module(f, [d])
Doug Zongkereb0a78a2014-01-27 10:01:06 -08001014 print "loaded device-specific extensions from", path
Doug Zongker8e2f2b92009-06-24 14:34:57 -07001015 self.module = imp.load_module("device_specific", *info)
1016 except ImportError:
1017 print "unable to load device-specific module; assuming none"
Doug Zongker05d3dea2009-06-22 11:32:31 -07001018
1019 def _DoCall(self, function_name, *args, **kwargs):
1020 """Call the named function in the device-specific module, passing
1021 the given args and kwargs. The first argument to the call will be
1022 the DeviceSpecific object itself. If there is no module, or the
1023 module does not define the function, return the value of the
1024 'default' kwarg (which itself defaults to None)."""
1025 if self.module is None or not hasattr(self.module, function_name):
1026 return kwargs.get("default", None)
1027 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
1028
1029 def FullOTA_Assertions(self):
1030 """Called after emitting the block of assertions at the top of a
1031 full OTA package. Implementations can add whatever additional
1032 assertions they like."""
1033 return self._DoCall("FullOTA_Assertions")
1034
Doug Zongkere5ff5902012-01-17 10:55:37 -08001035 def FullOTA_InstallBegin(self):
1036 """Called at the start of full OTA installation."""
1037 return self._DoCall("FullOTA_InstallBegin")
1038
Doug Zongker05d3dea2009-06-22 11:32:31 -07001039 def FullOTA_InstallEnd(self):
1040 """Called at the end of full OTA installation; typically this is
1041 used to install the image for the device's baseband processor."""
1042 return self._DoCall("FullOTA_InstallEnd")
1043
1044 def IncrementalOTA_Assertions(self):
1045 """Called after emitting the block of assertions at the top of an
1046 incremental OTA package. Implementations can add whatever
1047 additional assertions they like."""
1048 return self._DoCall("IncrementalOTA_Assertions")
1049
Doug Zongkere5ff5902012-01-17 10:55:37 -08001050 def IncrementalOTA_VerifyBegin(self):
1051 """Called at the start of the verification phase of incremental
1052 OTA installation; additional checks can be placed here to abort
1053 the script before any changes are made."""
1054 return self._DoCall("IncrementalOTA_VerifyBegin")
1055
Doug Zongker05d3dea2009-06-22 11:32:31 -07001056 def IncrementalOTA_VerifyEnd(self):
1057 """Called at the end of the verification phase of incremental OTA
1058 installation; additional checks can be placed here to abort the
1059 script before any changes are made."""
1060 return self._DoCall("IncrementalOTA_VerifyEnd")
1061
Doug Zongkere5ff5902012-01-17 10:55:37 -08001062 def IncrementalOTA_InstallBegin(self):
1063 """Called at the start of incremental OTA installation (after
1064 verification is complete)."""
1065 return self._DoCall("IncrementalOTA_InstallBegin")
1066
Doug Zongker05d3dea2009-06-22 11:32:31 -07001067 def IncrementalOTA_InstallEnd(self):
1068 """Called at the end of incremental OTA installation; typically
1069 this is used to install the image for the device's baseband
1070 processor."""
1071 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001072
1073class File(object):
1074 def __init__(self, name, data):
1075 self.name = name
1076 self.data = data
1077 self.size = len(data)
Doug Zongker55d93282011-01-25 17:03:34 -08001078 self.sha1 = sha1(data).hexdigest()
1079
1080 @classmethod
1081 def FromLocalFile(cls, name, diskname):
1082 f = open(diskname, "rb")
1083 data = f.read()
1084 f.close()
1085 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001086
1087 def WriteToTemp(self):
1088 t = tempfile.NamedTemporaryFile()
1089 t.write(self.data)
1090 t.flush()
1091 return t
1092
Geremy Condra36bd3652014-02-06 19:45:10 -08001093 def AddToZip(self, z, compression=None):
Tao Baof3282b42015-04-01 11:21:55 -07001094 ZipWriteStr(z, self.name, self.data, compress_type=compression)
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001095
1096DIFF_PROGRAM_BY_EXT = {
1097 ".gz" : "imgdiff",
1098 ".zip" : ["imgdiff", "-z"],
1099 ".jar" : ["imgdiff", "-z"],
1100 ".apk" : ["imgdiff", "-z"],
1101 ".img" : "imgdiff",
1102 }
1103
1104class Difference(object):
Doug Zongker24cd2802012-08-14 16:36:15 -07001105 def __init__(self, tf, sf, diff_program=None):
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001106 self.tf = tf
1107 self.sf = sf
1108 self.patch = None
Doug Zongker24cd2802012-08-14 16:36:15 -07001109 self.diff_program = diff_program
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001110
1111 def ComputePatch(self):
1112 """Compute the patch (as a string of data) needed to turn sf into
1113 tf. Returns the same tuple as GetPatch()."""
1114
1115 tf = self.tf
1116 sf = self.sf
1117
Doug Zongker24cd2802012-08-14 16:36:15 -07001118 if self.diff_program:
1119 diff_program = self.diff_program
1120 else:
1121 ext = os.path.splitext(tf.name)[1]
1122 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001123
1124 ttemp = tf.WriteToTemp()
1125 stemp = sf.WriteToTemp()
1126
1127 ext = os.path.splitext(tf.name)[1]
1128
1129 try:
1130 ptemp = tempfile.NamedTemporaryFile()
1131 if isinstance(diff_program, list):
1132 cmd = copy.copy(diff_program)
1133 else:
1134 cmd = [diff_program]
1135 cmd.append(stemp.name)
1136 cmd.append(ttemp.name)
1137 cmd.append(ptemp.name)
1138 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Doug Zongkerf8340082014-08-05 10:39:37 -07001139 err = []
1140 def run():
1141 _, e = p.communicate()
Dan Albert8b72aef2015-03-23 19:13:21 -07001142 if e:
1143 err.append(e)
Doug Zongkerf8340082014-08-05 10:39:37 -07001144 th = threading.Thread(target=run)
1145 th.start()
1146 th.join(timeout=300) # 5 mins
1147 if th.is_alive():
1148 print "WARNING: diff command timed out"
1149 p.terminate()
1150 th.join(5)
1151 if th.is_alive():
1152 p.kill()
1153 th.join()
1154
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001155 if err or p.returncode != 0:
Doug Zongkerf8340082014-08-05 10:39:37 -07001156 print "WARNING: failure running %s:\n%s\n" % (
1157 diff_program, "".join(err))
1158 self.patch = None
1159 return None, None, None
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001160 diff = ptemp.read()
1161 finally:
1162 ptemp.close()
1163 stemp.close()
1164 ttemp.close()
1165
1166 self.patch = diff
1167 return self.tf, self.sf, self.patch
1168
1169
1170 def GetPatch(self):
1171 """Return a tuple (target_file, source_file, patch_data).
1172 patch_data may be None if ComputePatch hasn't been called, or if
1173 computing the patch failed."""
1174 return self.tf, self.sf, self.patch
1175
1176
1177def ComputeDifferences(diffs):
1178 """Call ComputePatch on all the Difference objects in 'diffs'."""
1179 print len(diffs), "diffs to compute"
1180
1181 # Do the largest files first, to try and reduce the long-pole effect.
1182 by_size = [(i.tf.size, i) for i in diffs]
1183 by_size.sort(reverse=True)
1184 by_size = [i[1] for i in by_size]
1185
1186 lock = threading.Lock()
1187 diff_iter = iter(by_size) # accessed under lock
1188
1189 def worker():
1190 try:
1191 lock.acquire()
1192 for d in diff_iter:
1193 lock.release()
1194 start = time.time()
1195 d.ComputePatch()
1196 dur = time.time() - start
1197 lock.acquire()
1198
1199 tf, sf, patch = d.GetPatch()
1200 if sf.name == tf.name:
1201 name = tf.name
1202 else:
1203 name = "%s (%s)" % (tf.name, sf.name)
1204 if patch is None:
1205 print "patching failed! %s" % (name,)
1206 else:
1207 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
1208 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
1209 lock.release()
Dan Albert8b72aef2015-03-23 19:13:21 -07001210 except Exception as e:
Doug Zongkerea5d7a92010-09-12 15:26:16 -07001211 print e
1212 raise
1213
1214 # start worker threads; wait for them all to finish.
1215 threads = [threading.Thread(target=worker)
1216 for i in range(OPTIONS.worker_threads)]
1217 for th in threads:
1218 th.start()
1219 while threads:
1220 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -07001221
1222
Dan Albert8b72aef2015-03-23 19:13:21 -07001223class BlockDifference(object):
1224 def __init__(self, partition, tgt, src=None, check_first_block=False,
1225 version=None):
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001226 self.tgt = tgt
1227 self.src = src
1228 self.partition = partition
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001229 self.check_first_block = check_first_block
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001230
Tao Bao5ece99d2015-05-12 11:42:31 -07001231 # Due to http://b/20939131, check_first_block is disabled temporarily.
1232 assert not self.check_first_block
1233
Tao Baodd2a5892015-03-12 12:32:37 -07001234 if version is None:
1235 version = 1
1236 if OPTIONS.info_dict:
1237 version = max(
1238 int(i) for i in
1239 OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(","))
1240 self.version = version
Doug Zongker62338182014-09-08 08:29:55 -07001241
1242 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads,
Michael Runge910b0052015-02-11 19:28:08 -08001243 version=self.version)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001244 tmpdir = tempfile.mkdtemp()
1245 OPTIONS.tempfiles.append(tmpdir)
1246 self.path = os.path.join(tmpdir, partition)
1247 b.Compute(self.path)
1248
Tao Baoe09359a2015-10-13 16:37:12 -07001249 if src is None:
1250 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)
1251 else:
1252 _, self.device = GetTypeAndDevice("/" + partition,
1253 OPTIONS.source_info_dict)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001254
1255 def WriteScript(self, script, output_zip, progress=None):
1256 if not self.src:
1257 # write the output unconditionally
Jesse Zhao75bcea02015-01-06 10:59:53 -08001258 script.Print("Patching %s image unconditionally..." % (self.partition,))
1259 else:
1260 script.Print("Patching %s image after verification." % (self.partition,))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001261
Dan Albert8b72aef2015-03-23 19:13:21 -07001262 if progress:
1263 script.ShowProgress(progress, 0)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001264 self._WriteUpdate(script, output_zip)
Tao Bao5fcaaef2015-06-01 13:40:49 -07001265 self._WritePostInstallVerifyScript(script)
Jesse Zhao75bcea02015-01-06 10:59:53 -08001266
1267 def WriteVerifyScript(self, script):
Sami Tolvanendd67a292014-12-09 16:40:34 +00001268 partition = self.partition
Jesse Zhao75bcea02015-01-06 10:59:53 -08001269 if not self.src:
Sami Tolvanendd67a292014-12-09 16:40:34 +00001270 script.Print("Image %s will be patched unconditionally." % (partition,))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001271 else:
Tao Bao5ece99d2015-05-12 11:42:31 -07001272 ranges = self.src.care_map.subtract(self.src.clobbered_blocks)
1273 ranges_str = ranges.to_string_raw()
Michael Runge910b0052015-02-11 19:28:08 -08001274 if self.version >= 3:
Sami Tolvanene09d0962015-04-24 11:54:01 +01001275 script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" || '
1276 'block_image_verify("%s", '
Michael Runge910b0052015-02-11 19:28:08 -08001277 'package_extract_file("%s.transfer.list"), '
Sami Tolvanene09d0962015-04-24 11:54:01 +01001278 '"%s.new.dat", "%s.patch.dat")) then') % (
Tao Bao5ece99d2015-05-12 11:42:31 -07001279 self.device, ranges_str, self.src.TotalSha1(),
Sami Tolvanene09d0962015-04-24 11:54:01 +01001280 self.device, partition, partition, partition))
Michael Runge910b0052015-02-11 19:28:08 -08001281 else:
Dan Albert8b72aef2015-03-23 19:13:21 -07001282 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
Tao Bao5ece99d2015-05-12 11:42:31 -07001283 self.device, ranges_str, self.src.TotalSha1()))
Tao Baodd2a5892015-03-12 12:32:37 -07001284 script.Print('Verified %s image...' % (partition,))
Dan Albert8b72aef2015-03-23 19:13:21 -07001285 script.AppendExtra('else')
Sami Tolvanendd67a292014-12-09 16:40:34 +00001286
Tao Baodd2a5892015-03-12 12:32:37 -07001287 # When generating incrementals for the system and vendor partitions,
1288 # explicitly check the first block (which contains the superblock) of
1289 # the partition to see if it's what we expect. If this check fails,
1290 # give an explicit log message about the partition having been
1291 # remounted R/W (the most likely explanation) and the need to flash to
1292 # get OTAs working again.
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001293 if self.check_first_block:
1294 self._CheckFirstBlock(script)
1295
Tao Baodd2a5892015-03-12 12:32:37 -07001296 # Abort the OTA update. Note that the incremental OTA cannot be applied
1297 # even if it may match the checksum of the target partition.
1298 # a) If version < 3, operations like move and erase will make changes
1299 # unconditionally and damage the partition.
1300 # b) If version >= 3, it won't even reach here.
1301 script.AppendExtra(('abort("%s partition has unexpected contents");\n'
1302 'endif;') % (partition,))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001303
Tao Bao5fcaaef2015-06-01 13:40:49 -07001304 def _WritePostInstallVerifyScript(self, script):
1305 partition = self.partition
1306 script.Print('Verifying the updated %s image...' % (partition,))
1307 # Unlike pre-install verification, clobbered_blocks should not be ignored.
1308 ranges = self.tgt.care_map
1309 ranges_str = ranges.to_string_raw()
1310 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
1311 self.device, ranges_str,
1312 self.tgt.TotalSha1(include_clobbered_blocks=True)))
Tao Bao2fd2c9b2015-07-09 17:37:49 -07001313
1314 # Bug: 20881595
1315 # Verify that extended blocks are really zeroed out.
1316 if self.tgt.extended:
1317 ranges_str = self.tgt.extended.to_string_raw()
1318 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
1319 self.device, ranges_str,
1320 self._HashZeroBlocks(self.tgt.extended.size())))
1321 script.Print('Verified the updated %s image.' % (partition,))
1322 script.AppendExtra(
1323 'else\n'
1324 ' abort("%s partition has unexpected non-zero contents after OTA '
1325 'update");\n'
1326 'endif;' % (partition,))
1327 else:
1328 script.Print('Verified the updated %s image.' % (partition,))
1329
Tao Bao5fcaaef2015-06-01 13:40:49 -07001330 script.AppendExtra(
1331 'else\n'
1332 ' abort("%s partition has unexpected contents after OTA update");\n'
1333 'endif;' % (partition,))
1334
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001335 def _WriteUpdate(self, script, output_zip):
Dan Albert8e0178d2015-01-27 15:53:15 -08001336 ZipWrite(output_zip,
1337 '{}.transfer.list'.format(self.path),
1338 '{}.transfer.list'.format(self.partition))
1339 ZipWrite(output_zip,
1340 '{}.new.dat'.format(self.path),
1341 '{}.new.dat'.format(self.partition))
1342 ZipWrite(output_zip,
1343 '{}.patch.dat'.format(self.path),
1344 '{}.patch.dat'.format(self.partition),
1345 compress_type=zipfile.ZIP_STORED)
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001346
Dan Albert8e0178d2015-01-27 15:53:15 -08001347 call = ('block_image_update("{device}", '
1348 'package_extract_file("{partition}.transfer.list"), '
1349 '"{partition}.new.dat", "{partition}.patch.dat");\n'.format(
1350 device=self.device, partition=self.partition))
Dan Albert8b72aef2015-03-23 19:13:21 -07001351 script.AppendExtra(script.WordWrap(call))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001352
Dan Albert8b72aef2015-03-23 19:13:21 -07001353 def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use
Sami Tolvanendd67a292014-12-09 16:40:34 +00001354 data = source.ReadRangeSet(ranges)
1355 ctx = sha1()
1356
1357 for p in data:
1358 ctx.update(p)
1359
1360 return ctx.hexdigest()
1361
Tao Bao2fd2c9b2015-07-09 17:37:49 -07001362 def _HashZeroBlocks(self, num_blocks): # pylint: disable=no-self-use
1363 """Return the hash value for all zero blocks."""
1364 zero_block = '\x00' * 4096
1365 ctx = sha1()
1366 for _ in range(num_blocks):
1367 ctx.update(zero_block)
1368
1369 return ctx.hexdigest()
1370
Tao Bao5ece99d2015-05-12 11:42:31 -07001371 # TODO(tbao): Due to http://b/20939131, block 0 may be changed without
1372 # remounting R/W. Will change the checking to a finer-grained way to
1373 # mask off those bits.
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001374 def _CheckFirstBlock(self, script):
Dan Albert8b72aef2015-03-23 19:13:21 -07001375 r = rangelib.RangeSet((0, 1))
1376 srchash = self._HashBlocks(self.src, r)
Doug Zongkerb34fcce2014-09-11 09:34:56 -07001377
1378 script.AppendExtra(('(range_sha1("%s", "%s") == "%s") || '
1379 'abort("%s has been remounted R/W; '
1380 'reflash device to reenable OTA updates");')
Sami Tolvanendd67a292014-12-09 16:40:34 +00001381 % (self.device, r.to_string_raw(), srchash,
Sami Tolvanendd67a292014-12-09 16:40:34 +00001382 self.device))
Doug Zongkerab7ca1d2014-08-26 10:40:28 -07001383
1384DataImage = blockimgdiff.DataImage
1385
1386
Doug Zongker96a57e72010-09-26 14:57:41 -07001387# map recovery.fstab's fs_types to mount/format "partition types"
Dan Albert8b72aef2015-03-23 19:13:21 -07001388PARTITION_TYPES = {
1389 "yaffs2": "MTD",
1390 "mtd": "MTD",
1391 "ext4": "EMMC",
1392 "emmc": "EMMC",
Mohamad Ayyash95e74c12015-05-01 15:39:36 -07001393 "f2fs": "EMMC",
1394 "squashfs": "EMMC"
Dan Albert8b72aef2015-03-23 19:13:21 -07001395}
Doug Zongker96a57e72010-09-26 14:57:41 -07001396
1397def GetTypeAndDevice(mount_point, info):
1398 fstab = info["fstab"]
1399 if fstab:
Dan Albert8b72aef2015-03-23 19:13:21 -07001400 return (PARTITION_TYPES[fstab[mount_point].fs_type],
1401 fstab[mount_point].device)
Doug Zongker96a57e72010-09-26 14:57:41 -07001402 else:
Dan Albert8b72aef2015-03-23 19:13:21 -07001403 raise KeyError
Baligh Uddinbeb6afd2013-11-13 00:22:34 +00001404
1405
1406def ParseCertificate(data):
1407 """Parse a PEM-format certificate."""
1408 cert = []
1409 save = False
1410 for line in data.split("\n"):
1411 if "--END CERTIFICATE--" in line:
1412 break
1413 if save:
1414 cert.append(line)
1415 if "--BEGIN CERTIFICATE--" in line:
1416 save = True
1417 cert = "".join(cert).decode('base64')
1418 return cert
Doug Zongkerc9253822014-02-04 12:17:58 -08001419
Doug Zongker412c02f2014-02-13 10:58:24 -08001420def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img,
1421 info_dict=None):
Doug Zongkerc9253822014-02-04 12:17:58 -08001422 """Generate a binary patch that creates the recovery image starting
1423 with the boot image. (Most of the space in these images is just the
1424 kernel, which is identical for the two, so the resulting patch
1425 should be efficient.) Add it to the output zip, along with a shell
1426 script that is run from init.rc on first boot to actually do the
1427 patching and install the new recovery image.
1428
1429 recovery_img and boot_img should be File objects for the
1430 corresponding images. info should be the dictionary returned by
1431 common.LoadInfoDict() on the input target_files.
1432 """
1433
Doug Zongker412c02f2014-02-13 10:58:24 -08001434 if info_dict is None:
1435 info_dict = OPTIONS.info_dict
1436
Tao Bao6ed14912015-07-22 12:33:18 -07001437 full_recovery_image = info_dict.get("full_recovery_image", None) == "true"
Tao Baob11d2c52015-07-21 18:01:20 -07001438 system_root_image = info_dict.get("system_root_image", None) == "true"
Doug Zongkerc9253822014-02-04 12:17:58 -08001439
Tao Bao6ed14912015-07-22 12:33:18 -07001440 if full_recovery_image:
1441 output_sink("etc/recovery.img", recovery_img.data)
1442
1443 else:
1444 diff_program = ["imgdiff"]
1445 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat")
1446 if os.path.exists(path):
1447 diff_program.append("-b")
1448 diff_program.append(path)
1449 bonus_args = "-b /system/etc/recovery-resource.dat"
1450 else:
1451 bonus_args = ""
1452
1453 d = Difference(recovery_img, boot_img, diff_program=diff_program)
1454 _, _, patch = d.ComputePatch()
1455 output_sink("recovery-from-boot.p", patch)
Doug Zongkerc9253822014-02-04 12:17:58 -08001456
Dan Albertebb19aa2015-03-27 19:11:53 -07001457 try:
Tao Baoe09359a2015-10-13 16:37:12 -07001458 # The following GetTypeAndDevice()s need to use the path in the target
1459 # info_dict instead of source_info_dict.
Dan Albertebb19aa2015-03-27 19:11:53 -07001460 boot_type, boot_device = GetTypeAndDevice("/boot", info_dict)
1461 recovery_type, recovery_device = GetTypeAndDevice("/recovery", info_dict)
1462 except KeyError:
Ying Wanga961a092014-07-29 11:42:37 -07001463 return
Doug Zongkerc9253822014-02-04 12:17:58 -08001464
Tao Bao6ed14912015-07-22 12:33:18 -07001465 if full_recovery_image:
1466 sh = """#!/system/bin/sh
1467if ! applypatch -c %(type)s:%(device)s:%(size)d:%(sha1)s; then
1468 applypatch /system/etc/recovery.img %(type)s:%(device)s %(sha1)s %(size)d && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed"
1469else
1470 log -t recovery "Recovery image already installed"
1471fi
1472""" % {'type': recovery_type,
1473 'device': recovery_device,
1474 'sha1': recovery_img.sha1,
1475 'size': recovery_img.size}
1476 else:
1477 sh = """#!/system/bin/sh
Doug Zongkerc9253822014-02-04 12:17:58 -08001478if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then
1479 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed"
1480else
1481 log -t recovery "Recovery image already installed"
1482fi
Dan Albert8b72aef2015-03-23 19:13:21 -07001483""" % {'boot_size': boot_img.size,
1484 'boot_sha1': boot_img.sha1,
1485 'recovery_size': recovery_img.size,
1486 'recovery_sha1': recovery_img.sha1,
1487 'boot_type': boot_type,
1488 'boot_device': boot_device,
1489 'recovery_type': recovery_type,
1490 'recovery_device': recovery_device,
1491 'bonus_args': bonus_args}
Doug Zongkerc9253822014-02-04 12:17:58 -08001492
1493 # The install script location moved from /system/etc to /system/bin
Tao Bao610754e2015-07-07 18:31:47 -07001494 # in the L release. Parse init.*.rc files to find out where the
Doug Zongkerc9253822014-02-04 12:17:58 -08001495 # target-files expects it to be, and put it there.
1496 sh_location = "etc/install-recovery.sh"
Tao Bao610754e2015-07-07 18:31:47 -07001497 found = False
Tao Baob11d2c52015-07-21 18:01:20 -07001498 if system_root_image:
1499 init_rc_dir = os.path.join(input_dir, "ROOT")
1500 else:
1501 init_rc_dir = os.path.join(input_dir, "BOOT", "RAMDISK")
Tao Bao610754e2015-07-07 18:31:47 -07001502 init_rc_files = os.listdir(init_rc_dir)
1503 for init_rc_file in init_rc_files:
1504 if (not init_rc_file.startswith('init.') or
1505 not init_rc_file.endswith('.rc')):
1506 continue
Tao Bao610754e2015-07-07 18:31:47 -07001507 with open(os.path.join(init_rc_dir, init_rc_file)) as f:
Doug Zongkerc9253822014-02-04 12:17:58 -08001508 for line in f:
Dan Albert8b72aef2015-03-23 19:13:21 -07001509 m = re.match(r"^service flash_recovery /system/(\S+)\s*$", line)
Doug Zongkerc9253822014-02-04 12:17:58 -08001510 if m:
1511 sh_location = m.group(1)
Tao Bao610754e2015-07-07 18:31:47 -07001512 found = True
Doug Zongkerc9253822014-02-04 12:17:58 -08001513 break
Tao Bao610754e2015-07-07 18:31:47 -07001514 if found:
1515 break
Tao Bao610754e2015-07-07 18:31:47 -07001516 print "putting script in", sh_location
Doug Zongkerc9253822014-02-04 12:17:58 -08001517
1518 output_sink(sh_location, sh)