blob: 2cb5680ed94bb72e0757be4176df2c3f14468756 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Doug Zongkerea5d7a92010-09-12 15:26:16 -070015import copy
Doug Zongker8ce7c252009-05-22 13:34:54 -070016import errno
Doug Zongkereef39442009-04-02 12:14:19 -070017import getopt
18import getpass
Doug Zongker05d3dea2009-06-22 11:32:31 -070019import imp
Doug Zongkereef39442009-04-02 12:14:19 -070020import os
Ying Wang7e6d4e42010-12-13 16:25:36 -080021import platform
Doug Zongkereef39442009-04-02 12:14:19 -070022import re
23import shutil
24import subprocess
25import sys
26import tempfile
Doug Zongkerea5d7a92010-09-12 15:26:16 -070027import threading
28import time
Doug Zongker048e7ca2009-06-15 14:31:53 -070029import zipfile
Doug Zongkereef39442009-04-02 12:14:19 -070030
Doug Zongker55d93282011-01-25 17:03:34 -080031try:
davidcad0bb92011-03-15 14:21:38 +000032 from hashlib import sha1 as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080033except ImportError:
davidcad0bb92011-03-15 14:21:38 +000034 from sha import sha as sha1
Doug Zongker55d93282011-01-25 17:03:34 -080035
Doug Zongkereef39442009-04-02 12:14:19 -070036# missing in Python 2.4 and before
37if not hasattr(os, "SEEK_SET"):
38 os.SEEK_SET = 0
39
40class Options(object): pass
41OPTIONS = Options()
Doug Zongker602a84e2009-06-18 08:35:12 -070042OPTIONS.search_path = "out/host/linux-x86"
Doug Zongkereef39442009-04-02 12:14:19 -070043OPTIONS.verbose = False
44OPTIONS.tempfiles = []
Doug Zongker05d3dea2009-06-22 11:32:31 -070045OPTIONS.device_specific = None
Doug Zongker8bec09e2009-11-30 15:37:14 -080046OPTIONS.extras = {}
Doug Zongkerc77a9ad2010-09-16 11:28:43 -070047OPTIONS.info_dict = None
Doug Zongkereef39442009-04-02 12:14:19 -070048
Doug Zongkerf6a53aa2009-12-15 15:06:55 -080049
50# Values for "certificate" in apkcerts that mean special things.
51SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
52
53
Doug Zongkereef39442009-04-02 12:14:19 -070054class ExternalError(RuntimeError): pass
55
56
57def Run(args, **kwargs):
58 """Create and return a subprocess.Popen object, printing the command
59 line on the terminal if -v was specified."""
60 if OPTIONS.verbose:
61 print " running: ", " ".join(args)
62 return subprocess.Popen(args, **kwargs)
63
64
Ying Wang7e6d4e42010-12-13 16:25:36 -080065def CloseInheritedPipes():
66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
67 before doing other work."""
68 if platform.system() != "Darwin":
69 return
70 for d in range(3, 1025):
71 try:
72 stat = os.fstat(d)
73 if stat is not None:
74 pipebit = stat[0] & 0x1000
75 if pipebit != 0:
76 os.close(d)
77 except OSError:
78 pass
79
80
Doug Zongker37974732010-09-16 17:44:38 -070081def LoadInfoDict(zip):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070082 """Read and parse the META/misc_info.txt key/value pairs from the
83 input target files and return a dict."""
84
85 d = {}
86 try:
Doug Zongker37974732010-09-16 17:44:38 -070087 for line in zip.read("META/misc_info.txt").split("\n"):
Doug Zongkerc19a8d52010-07-01 15:30:11 -070088 line = line.strip()
89 if not line or line.startswith("#"): continue
90 k, v = line.split("=", 1)
91 d[k] = v
Doug Zongker37974732010-09-16 17:44:38 -070092 except KeyError:
93 # ok if misc_info.txt doesn't exist
94 pass
Doug Zongkerc19a8d52010-07-01 15:30:11 -070095
Doug Zongker37974732010-09-16 17:44:38 -070096 # backwards compatibility: These values used to be in their own
97 # files. Look for them, in case we're processing an old
98 # target_files zip.
99
100 if "mkyaffs2_extra_flags" not in d:
101 try:
102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
103 except KeyError:
104 # ok if flags don't exist
105 pass
106
107 if "recovery_api_version" not in d:
108 try:
109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
110 except KeyError:
111 raise ValueError("can't find recovery API version in input target-files")
112
113 if "tool_extensions" not in d:
114 try:
115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
116 except KeyError:
117 # ok if extensions don't exist
118 pass
119
120 try:
121 data = zip.read("META/imagesizes.txt")
122 for line in data.split("\n"):
123 if not line: continue
Doug Zongker1684d9c2010-09-17 07:44:38 -0700124 name, value = line.split(" ", 1)
125 if not value: continue
Doug Zongker37974732010-09-16 17:44:38 -0700126 if name == "blocksize":
127 d[name] = value
128 else:
129 d[name + "_size"] = value
130 except KeyError:
131 pass
132
133 def makeint(key):
134 if key in d:
135 d[key] = int(d[key], 0)
136
137 makeint("recovery_api_version")
138 makeint("blocksize")
139 makeint("system_size")
140 makeint("userdata_size")
141 makeint("recovery_size")
142 makeint("boot_size")
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700143
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700144 d["fstab"] = LoadRecoveryFSTab(zip)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700145 return d
146
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700147def LoadRecoveryFSTab(zip):
148 class Partition(object):
149 pass
150
151 try:
152 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
153 except KeyError:
Ying Wanga73b6562011-03-03 21:52:08 -0800154 raise ValueError("Could not find RECOVERY/RAMDISK/etc/recovery.fstab")
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700155
156 d = {}
157 for line in data.split("\n"):
158 line = line.strip()
159 if not line or line.startswith("#"): continue
160 pieces = line.split()
161 if not (3 <= len(pieces) <= 4):
162 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
163
164 p = Partition()
165 p.mount_point = pieces[0]
166 p.fs_type = pieces[1]
167 p.device = pieces[2]
Doug Zongker086cbb02011-02-17 15:54:20 -0800168 p.length = 0
169 options = None
170 if len(pieces) >= 4:
171 if pieces[3].startswith("/"):
172 p.device2 = pieces[3]
173 if len(pieces) >= 5:
174 options = pieces[4]
175 else:
176 p.device2 = None
177 options = pieces[3]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700178 else:
179 p.device2 = None
180
Doug Zongker086cbb02011-02-17 15:54:20 -0800181 if options:
182 options = options.split(",")
183 for i in options:
184 if i.startswith("length="):
185 p.length = int(i[7:])
186 else:
187 print "%s: unknown option \"%s\"" % (p.mount_point, i)
188
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700189 d[p.mount_point] = p
190 return d
191
192
Doug Zongker37974732010-09-16 17:44:38 -0700193def DumpInfoDict(d):
194 for k, v in sorted(d.items()):
195 print "%-25s = (%s) %s" % (k, type(v).__name__, v)
Doug Zongkerc19a8d52010-07-01 15:30:11 -0700196
Doug Zongkereef39442009-04-02 12:14:19 -0700197def BuildBootableImage(sourcedir):
198 """Take a kernel, cmdline, and ramdisk directory from the input (in
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700199 'sourcedir'), and turn them into a boot image. Return the image
200 data, or None if sourcedir does not appear to contains files for
201 building the requested image."""
202
203 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
204 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
205 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700206
207 ramdisk_img = tempfile.NamedTemporaryFile()
208 img = tempfile.NamedTemporaryFile()
209
210 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
211 stdout=subprocess.PIPE)
Doug Zongker32da27a2009-05-29 09:35:56 -0700212 p2 = Run(["minigzip"],
213 stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
Doug Zongkereef39442009-04-02 12:14:19 -0700214
215 p2.wait()
216 p1.wait()
217 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
Doug Zongker32da27a2009-05-29 09:35:56 -0700218 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
Doug Zongkereef39442009-04-02 12:14:19 -0700219
Doug Zongker38a649f2009-06-17 09:07:09 -0700220 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
221
Doug Zongker171f1cd2009-06-15 22:36:37 -0700222 fn = os.path.join(sourcedir, "cmdline")
223 if os.access(fn, os.F_OK):
Doug Zongker38a649f2009-06-17 09:07:09 -0700224 cmd.append("--cmdline")
225 cmd.append(open(fn).read().rstrip("\n"))
226
227 fn = os.path.join(sourcedir, "base")
228 if os.access(fn, os.F_OK):
229 cmd.append("--base")
230 cmd.append(open(fn).read().rstrip("\n"))
231
Ying Wang4de6b5b2010-08-25 14:29:34 -0700232 fn = os.path.join(sourcedir, "pagesize")
233 if os.access(fn, os.F_OK):
234 cmd.append("--pagesize")
235 cmd.append(open(fn).read().rstrip("\n"))
236
Doug Zongker38a649f2009-06-17 09:07:09 -0700237 cmd.extend(["--ramdisk", ramdisk_img.name,
238 "--output", img.name])
239
240 p = Run(cmd, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700241 p.communicate()
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700242 assert p.returncode == 0, "mkbootimg of %s image failed" % (
243 os.path.basename(sourcedir),)
Doug Zongkereef39442009-04-02 12:14:19 -0700244
245 img.seek(os.SEEK_SET, 0)
246 data = img.read()
247
248 ramdisk_img.close()
249 img.close()
250
251 return data
252
253
Doug Zongker55d93282011-01-25 17:03:34 -0800254def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
255 """Return a File object (with name 'name') with the desired bootable
256 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
257 'prebuilt_name', otherwise construct it from the source files in
258 'unpack_dir'/'tree_subdir'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700259
Doug Zongker55d93282011-01-25 17:03:34 -0800260 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
261 if os.path.exists(prebuilt_path):
262 print "using prebuilt %s..." % (prebuilt_name,)
263 return File.FromLocalFile(name, prebuilt_path)
264 else:
265 print "building image from target_files %s..." % (tree_subdir,)
266 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
267
Doug Zongkereef39442009-04-02 12:14:19 -0700268
Doug Zongker75f17362009-12-08 13:46:44 -0800269def UnzipTemp(filename, pattern=None):
Doug Zongker55d93282011-01-25 17:03:34 -0800270 """Unzip the given archive into a temporary directory and return the name.
271
272 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
273 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
274
275 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
276 main file), open for reading.
277 """
Doug Zongkereef39442009-04-02 12:14:19 -0700278
279 tmp = tempfile.mkdtemp(prefix="targetfiles-")
280 OPTIONS.tempfiles.append(tmp)
Doug Zongker55d93282011-01-25 17:03:34 -0800281
282 def unzip_to_dir(filename, dirname):
283 cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
284 if pattern is not None:
285 cmd.append(pattern)
286 p = Run(cmd, stdout=subprocess.PIPE)
287 p.communicate()
288 if p.returncode != 0:
289 raise ExternalError("failed to unzip input target-files \"%s\"" %
290 (filename,))
291
292 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
293 if m:
294 unzip_to_dir(m.group(1), tmp)
295 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
296 filename = m.group(1)
297 else:
298 unzip_to_dir(filename, tmp)
299
300 return tmp, zipfile.ZipFile(filename, "r")
Doug Zongkereef39442009-04-02 12:14:19 -0700301
302
303def GetKeyPasswords(keylist):
304 """Given a list of keys, prompt the user to enter passwords for
305 those which require them. Return a {key: password} dict. password
306 will be None if the key has no password."""
307
Doug Zongker8ce7c252009-05-22 13:34:54 -0700308 no_passwords = []
309 need_passwords = []
Doug Zongkereef39442009-04-02 12:14:19 -0700310 devnull = open("/dev/null", "w+b")
311 for k in sorted(keylist):
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800312 # We don't need a password for things that aren't really keys.
313 if k in SPECIAL_CERT_STRINGS:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700314 no_passwords.append(k)
Doug Zongker43874f82009-04-14 14:05:15 -0700315 continue
316
Doug Zongker602a84e2009-06-18 08:35:12 -0700317 p = Run(["openssl", "pkcs8", "-in", k+".pk8",
318 "-inform", "DER", "-nocrypt"],
319 stdin=devnull.fileno(),
320 stdout=devnull.fileno(),
321 stderr=subprocess.STDOUT)
Doug Zongkereef39442009-04-02 12:14:19 -0700322 p.communicate()
323 if p.returncode == 0:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700324 no_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700325 else:
Doug Zongker8ce7c252009-05-22 13:34:54 -0700326 need_passwords.append(k)
Doug Zongkereef39442009-04-02 12:14:19 -0700327 devnull.close()
Doug Zongker8ce7c252009-05-22 13:34:54 -0700328
329 key_passwords = PasswordManager().GetPasswords(need_passwords)
330 key_passwords.update(dict.fromkeys(no_passwords, None))
Doug Zongkereef39442009-04-02 12:14:19 -0700331 return key_passwords
332
333
Doug Zongker951495f2009-08-14 12:44:19 -0700334def SignFile(input_name, output_name, key, password, align=None,
335 whole_file=False):
Doug Zongkereef39442009-04-02 12:14:19 -0700336 """Sign the input_name zip/jar/apk, producing output_name. Use the
337 given key and password (the latter may be None if the key does not
338 have a password.
339
340 If align is an integer > 1, zipalign is run to align stored files in
341 the output zip on 'align'-byte boundaries.
Doug Zongker951495f2009-08-14 12:44:19 -0700342
343 If whole_file is true, use the "-w" option to SignApk to embed a
344 signature that covers the whole file in the archive comment of the
345 zip file.
Doug Zongkereef39442009-04-02 12:14:19 -0700346 """
Doug Zongker951495f2009-08-14 12:44:19 -0700347
Doug Zongkereef39442009-04-02 12:14:19 -0700348 if align == 0 or align == 1:
349 align = None
350
351 if align:
352 temp = tempfile.NamedTemporaryFile()
353 sign_name = temp.name
354 else:
355 sign_name = output_name
356
Doug Zongkerca855e92011-02-23 09:25:01 -0800357 cmd = ["java", "-Xmx2048m", "-jar",
Doug Zongker951495f2009-08-14 12:44:19 -0700358 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
359 if whole_file:
360 cmd.append("-w")
361 cmd.extend([key + ".x509.pem", key + ".pk8",
362 input_name, sign_name])
363
364 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
Doug Zongkereef39442009-04-02 12:14:19 -0700365 if password is not None:
366 password += "\n"
367 p.communicate(password)
368 if p.returncode != 0:
369 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
370
371 if align:
Doug Zongker602a84e2009-06-18 08:35:12 -0700372 p = Run(["zipalign", "-f", str(align), sign_name, output_name])
Doug Zongkereef39442009-04-02 12:14:19 -0700373 p.communicate()
374 if p.returncode != 0:
375 raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
376 temp.close()
377
378
Doug Zongker37974732010-09-16 17:44:38 -0700379def CheckSize(data, target, info_dict):
Doug Zongkereef39442009-04-02 12:14:19 -0700380 """Check the data string passed against the max size limit, if
381 any, for the given target. Raise exception if the data is too big.
382 Print a warning if the data is nearing the maximum size."""
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700383
Doug Zongker1684d9c2010-09-17 07:44:38 -0700384 if target.endswith(".img"): target = target[:-4]
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700385 mount_point = "/" + target
386
387 if info_dict["fstab"]:
388 if mount_point == "/userdata": mount_point = "/data"
389 p = info_dict["fstab"][mount_point]
390 fs_type = p.fs_type
391 limit = info_dict.get(p.device + "_size", None)
Doug Zongker9ce0fb62010-09-20 18:04:41 -0700392 if not fs_type or not limit: return
Doug Zongkereef39442009-04-02 12:14:19 -0700393
Doug Zongkerc77a9ad2010-09-16 11:28:43 -0700394 if fs_type == "yaffs2":
395 # image size should be increased by 1/64th to account for the
396 # spare area (64 bytes per 2k page)
397 limit = limit / 2048 * (2048+64)
Doug Zongker486de122010-09-16 14:01:56 -0700398 size = len(data)
399 pct = float(size) * 100.0 / limit
400 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
401 if pct >= 99.0:
402 raise ExternalError(msg)
403 elif pct >= 95.0:
404 print
405 print " WARNING: ", msg
406 print
407 elif OPTIONS.verbose:
408 print " ", msg
Doug Zongkereef39442009-04-02 12:14:19 -0700409
410
Doug Zongkerf6a53aa2009-12-15 15:06:55 -0800411def ReadApkCerts(tf_zip):
412 """Given a target_files ZipFile, parse the META/apkcerts.txt file
413 and return a {package: cert} dict."""
414 certmap = {}
415 for line in tf_zip.read("META/apkcerts.txt").split("\n"):
416 line = line.strip()
417 if not line: continue
418 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
419 r'private_key="(.*)"$', line)
420 if m:
421 name, cert, privkey = m.groups()
422 if cert in SPECIAL_CERT_STRINGS and not privkey:
423 certmap[name] = cert
424 elif (cert.endswith(".x509.pem") and
425 privkey.endswith(".pk8") and
426 cert[:-9] == privkey[:-4]):
427 certmap[name] = cert[:-9]
428 else:
429 raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
430 return certmap
431
432
Doug Zongkereef39442009-04-02 12:14:19 -0700433COMMON_DOCSTRING = """
434 -p (--path) <dir>
Doug Zongker602a84e2009-06-18 08:35:12 -0700435 Prepend <dir>/bin to the list of places to search for binaries
436 run by this script, and expect to find jars in <dir>/framework.
Doug Zongkereef39442009-04-02 12:14:19 -0700437
Doug Zongker05d3dea2009-06-22 11:32:31 -0700438 -s (--device_specific) <file>
439 Path to the python module containing device-specific
440 releasetools code.
441
Doug Zongker8bec09e2009-11-30 15:37:14 -0800442 -x (--extra) <key=value>
443 Add a key/value pair to the 'extras' dict, which device-specific
444 extension code may look at.
445
Doug Zongkereef39442009-04-02 12:14:19 -0700446 -v (--verbose)
447 Show command lines being executed.
448
449 -h (--help)
450 Display this usage message and exit.
451"""
452
453def Usage(docstring):
454 print docstring.rstrip("\n")
455 print COMMON_DOCSTRING
456
457
458def ParseOptions(argv,
459 docstring,
460 extra_opts="", extra_long_opts=(),
461 extra_option_handler=None):
462 """Parse the options in argv and return any arguments that aren't
463 flags. docstring is the calling module's docstring, to be displayed
464 for errors and -h. extra_opts and extra_long_opts are for flags
465 defined by the caller, which are processed by passing them to
466 extra_option_handler."""
467
468 try:
469 opts, args = getopt.getopt(
Doug Zongker8bec09e2009-11-30 15:37:14 -0800470 argv, "hvp:s:x:" + extra_opts,
471 ["help", "verbose", "path=", "device_specific=", "extra="] +
Doug Zongker05d3dea2009-06-22 11:32:31 -0700472 list(extra_long_opts))
Doug Zongkereef39442009-04-02 12:14:19 -0700473 except getopt.GetoptError, err:
474 Usage(docstring)
475 print "**", str(err), "**"
476 sys.exit(2)
477
478 path_specified = False
479
480 for o, a in opts:
481 if o in ("-h", "--help"):
482 Usage(docstring)
483 sys.exit()
484 elif o in ("-v", "--verbose"):
485 OPTIONS.verbose = True
486 elif o in ("-p", "--path"):
Doug Zongker602a84e2009-06-18 08:35:12 -0700487 OPTIONS.search_path = a
Doug Zongker05d3dea2009-06-22 11:32:31 -0700488 elif o in ("-s", "--device_specific"):
489 OPTIONS.device_specific = a
Doug Zongker5ecba702009-12-03 16:36:20 -0800490 elif o in ("-x", "--extra"):
Doug Zongker8bec09e2009-11-30 15:37:14 -0800491 key, value = a.split("=", 1)
492 OPTIONS.extras[key] = value
Doug Zongkereef39442009-04-02 12:14:19 -0700493 else:
494 if extra_option_handler is None or not extra_option_handler(o, a):
495 assert False, "unknown option \"%s\"" % (o,)
496
Doug Zongker602a84e2009-06-18 08:35:12 -0700497 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
498 os.pathsep + os.environ["PATH"])
Doug Zongkereef39442009-04-02 12:14:19 -0700499
500 return args
501
502
503def Cleanup():
504 for i in OPTIONS.tempfiles:
505 if os.path.isdir(i):
506 shutil.rmtree(i)
507 else:
508 os.remove(i)
Doug Zongker8ce7c252009-05-22 13:34:54 -0700509
510
511class PasswordManager(object):
512 def __init__(self):
513 self.editor = os.getenv("EDITOR", None)
514 self.pwfile = os.getenv("ANDROID_PW_FILE", None)
515
516 def GetPasswords(self, items):
517 """Get passwords corresponding to each string in 'items',
518 returning a dict. (The dict may have keys in addition to the
519 values in 'items'.)
520
521 Uses the passwords in $ANDROID_PW_FILE if available, letting the
522 user edit that file to add more needed passwords. If no editor is
523 available, or $ANDROID_PW_FILE isn't define, prompts the user
524 interactively in the ordinary way.
525 """
526
527 current = self.ReadFile()
528
529 first = True
530 while True:
531 missing = []
532 for i in items:
533 if i not in current or not current[i]:
534 missing.append(i)
535 # Are all the passwords already in the file?
536 if not missing: return current
537
538 for i in missing:
539 current[i] = ""
540
541 if not first:
542 print "key file %s still missing some passwords." % (self.pwfile,)
543 answer = raw_input("try to edit again? [y]> ").strip()
544 if answer and answer[0] not in 'yY':
545 raise RuntimeError("key passwords unavailable")
546 first = False
547
548 current = self.UpdateAndReadFile(current)
549
550 def PromptResult(self, current):
551 """Prompt the user to enter a value (password) for each key in
552 'current' whose value is fales. Returns a new dict with all the
553 values.
554 """
555 result = {}
556 for k, v in sorted(current.iteritems()):
557 if v:
558 result[k] = v
559 else:
560 while True:
561 result[k] = getpass.getpass("Enter password for %s key> "
562 % (k,)).strip()
563 if result[k]: break
564 return result
565
566 def UpdateAndReadFile(self, current):
567 if not self.editor or not self.pwfile:
568 return self.PromptResult(current)
569
570 f = open(self.pwfile, "w")
571 os.chmod(self.pwfile, 0600)
572 f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
573 f.write("# (Additional spaces are harmless.)\n\n")
574
575 first_line = None
576 sorted = [(not v, k, v) for (k, v) in current.iteritems()]
577 sorted.sort()
578 for i, (_, k, v) in enumerate(sorted):
579 f.write("[[[ %s ]]] %s\n" % (v, k))
580 if not v and first_line is None:
581 # position cursor on first line with no password.
582 first_line = i + 4
583 f.close()
584
585 p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
586 _, _ = p.communicate()
587
588 return self.ReadFile()
589
590 def ReadFile(self):
591 result = {}
592 if self.pwfile is None: return result
593 try:
594 f = open(self.pwfile, "r")
595 for line in f:
596 line = line.strip()
597 if not line or line[0] == '#': continue
598 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
599 if not m:
600 print "failed to parse password file: ", line
601 else:
602 result[m.group(2)] = m.group(1)
603 f.close()
604 except IOError, e:
605 if e.errno != errno.ENOENT:
606 print "error reading password file: ", str(e)
607 return result
Doug Zongker048e7ca2009-06-15 14:31:53 -0700608
609
610def ZipWriteStr(zip, filename, data, perms=0644):
611 # use a fixed timestamp so the output is repeatable.
612 zinfo = zipfile.ZipInfo(filename=filename,
613 date_time=(2009, 1, 1, 0, 0, 0))
614 zinfo.compress_type = zip.compression
615 zinfo.external_attr = perms << 16
616 zip.writestr(zinfo, data)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700617
618
619class DeviceSpecificParams(object):
620 module = None
621 def __init__(self, **kwargs):
622 """Keyword arguments to the constructor become attributes of this
623 object, which is passed to all functions in the device-specific
624 module."""
625 for k, v in kwargs.iteritems():
626 setattr(self, k, v)
Doug Zongker8bec09e2009-11-30 15:37:14 -0800627 self.extras = OPTIONS.extras
Doug Zongker05d3dea2009-06-22 11:32:31 -0700628
629 if self.module is None:
630 path = OPTIONS.device_specific
Doug Zongkerc18736b2009-09-30 09:20:32 -0700631 if not path: return
Doug Zongker8e2f2b92009-06-24 14:34:57 -0700632 try:
633 if os.path.isdir(path):
634 info = imp.find_module("releasetools", [path])
635 else:
636 d, f = os.path.split(path)
637 b, x = os.path.splitext(f)
638 if x == ".py":
639 f = b
640 info = imp.find_module(f, [d])
641 self.module = imp.load_module("device_specific", *info)
642 except ImportError:
643 print "unable to load device-specific module; assuming none"
Doug Zongker05d3dea2009-06-22 11:32:31 -0700644
645 def _DoCall(self, function_name, *args, **kwargs):
646 """Call the named function in the device-specific module, passing
647 the given args and kwargs. The first argument to the call will be
648 the DeviceSpecific object itself. If there is no module, or the
649 module does not define the function, return the value of the
650 'default' kwarg (which itself defaults to None)."""
651 if self.module is None or not hasattr(self.module, function_name):
652 return kwargs.get("default", None)
653 return getattr(self.module, function_name)(*((self,) + args), **kwargs)
654
655 def FullOTA_Assertions(self):
656 """Called after emitting the block of assertions at the top of a
657 full OTA package. Implementations can add whatever additional
658 assertions they like."""
659 return self._DoCall("FullOTA_Assertions")
660
661 def FullOTA_InstallEnd(self):
662 """Called at the end of full OTA installation; typically this is
663 used to install the image for the device's baseband processor."""
664 return self._DoCall("FullOTA_InstallEnd")
665
666 def IncrementalOTA_Assertions(self):
667 """Called after emitting the block of assertions at the top of an
668 incremental OTA package. Implementations can add whatever
669 additional assertions they like."""
670 return self._DoCall("IncrementalOTA_Assertions")
671
672 def IncrementalOTA_VerifyEnd(self):
673 """Called at the end of the verification phase of incremental OTA
674 installation; additional checks can be placed here to abort the
675 script before any changes are made."""
676 return self._DoCall("IncrementalOTA_VerifyEnd")
677
678 def IncrementalOTA_InstallEnd(self):
679 """Called at the end of incremental OTA installation; typically
680 this is used to install the image for the device's baseband
681 processor."""
682 return self._DoCall("IncrementalOTA_InstallEnd")
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700683
684class File(object):
685 def __init__(self, name, data):
686 self.name = name
687 self.data = data
688 self.size = len(data)
Doug Zongker55d93282011-01-25 17:03:34 -0800689 self.sha1 = sha1(data).hexdigest()
690
691 @classmethod
692 def FromLocalFile(cls, name, diskname):
693 f = open(diskname, "rb")
694 data = f.read()
695 f.close()
696 return File(name, data)
Doug Zongkerea5d7a92010-09-12 15:26:16 -0700697
698 def WriteToTemp(self):
699 t = tempfile.NamedTemporaryFile()
700 t.write(self.data)
701 t.flush()
702 return t
703
704 def AddToZip(self, z):
705 ZipWriteStr(z, self.name, self.data)
706
707DIFF_PROGRAM_BY_EXT = {
708 ".gz" : "imgdiff",
709 ".zip" : ["imgdiff", "-z"],
710 ".jar" : ["imgdiff", "-z"],
711 ".apk" : ["imgdiff", "-z"],
712 ".img" : "imgdiff",
713 }
714
715class Difference(object):
716 def __init__(self, tf, sf):
717 self.tf = tf
718 self.sf = sf
719 self.patch = None
720
721 def ComputePatch(self):
722 """Compute the patch (as a string of data) needed to turn sf into
723 tf. Returns the same tuple as GetPatch()."""
724
725 tf = self.tf
726 sf = self.sf
727
728 ext = os.path.splitext(tf.name)[1]
729 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
730
731 ttemp = tf.WriteToTemp()
732 stemp = sf.WriteToTemp()
733
734 ext = os.path.splitext(tf.name)[1]
735
736 try:
737 ptemp = tempfile.NamedTemporaryFile()
738 if isinstance(diff_program, list):
739 cmd = copy.copy(diff_program)
740 else:
741 cmd = [diff_program]
742 cmd.append(stemp.name)
743 cmd.append(ttemp.name)
744 cmd.append(ptemp.name)
745 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
746 _, err = p.communicate()
747 if err or p.returncode != 0:
748 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
749 return None
750 diff = ptemp.read()
751 finally:
752 ptemp.close()
753 stemp.close()
754 ttemp.close()
755
756 self.patch = diff
757 return self.tf, self.sf, self.patch
758
759
760 def GetPatch(self):
761 """Return a tuple (target_file, source_file, patch_data).
762 patch_data may be None if ComputePatch hasn't been called, or if
763 computing the patch failed."""
764 return self.tf, self.sf, self.patch
765
766
767def ComputeDifferences(diffs):
768 """Call ComputePatch on all the Difference objects in 'diffs'."""
769 print len(diffs), "diffs to compute"
770
771 # Do the largest files first, to try and reduce the long-pole effect.
772 by_size = [(i.tf.size, i) for i in diffs]
773 by_size.sort(reverse=True)
774 by_size = [i[1] for i in by_size]
775
776 lock = threading.Lock()
777 diff_iter = iter(by_size) # accessed under lock
778
779 def worker():
780 try:
781 lock.acquire()
782 for d in diff_iter:
783 lock.release()
784 start = time.time()
785 d.ComputePatch()
786 dur = time.time() - start
787 lock.acquire()
788
789 tf, sf, patch = d.GetPatch()
790 if sf.name == tf.name:
791 name = tf.name
792 else:
793 name = "%s (%s)" % (tf.name, sf.name)
794 if patch is None:
795 print "patching failed! %s" % (name,)
796 else:
797 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
798 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
799 lock.release()
800 except Exception, e:
801 print e
802 raise
803
804 # start worker threads; wait for them all to finish.
805 threads = [threading.Thread(target=worker)
806 for i in range(OPTIONS.worker_threads)]
807 for th in threads:
808 th.start()
809 while threads:
810 threads.pop().join()
Doug Zongker96a57e72010-09-26 14:57:41 -0700811
812
813# map recovery.fstab's fs_types to mount/format "partition types"
814PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
815 "ext4": "EMMC", "emmc": "EMMC" }
816
817def GetTypeAndDevice(mount_point, info):
818 fstab = info["fstab"]
819 if fstab:
820 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
821 else:
Ying Wanga73b6562011-03-03 21:52:08 -0800822 return None