blob: 38e3c8392cbc52eea9e9704a15d438e6c154a2cd [file] [log] [blame]
Doug Zongker02d444b2009-05-27 18:24:03 -07001/*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18 * This program constructs binary patches for images -- such as boot.img
19 * and recovery.img -- that consist primarily of large chunks of gzipped
20 * data interspersed with uncompressed data. Doing a naive bsdiff of
21 * these files is not useful because small changes in the data lead to
22 * large changes in the compressed bitstream; bsdiff patches of gzipped
23 * data are typically as large as the data itself.
24 *
25 * To patch these usefully, we break the source and target images up into
26 * chunks of two types: "normal" and "gzip". Normal chunks are simply
27 * patched using a plain bsdiff. Gzip chunks are first expanded, then a
28 * bsdiff is applied to the uncompressed data, then the patched data is
29 * gzipped using the same encoder parameters. Patched chunks are
30 * concatenated together to create the output file; the output image
31 * should be *exactly* the same series of bytes as the target image used
32 * originally to generate the patch.
33 *
34 * To work well with this tool, the gzipped sections of the target
35 * image must have been generated using the same deflate encoder that
36 * is available in applypatch, namely, the one in the zlib library.
37 * In practice this means that images should be compressed using the
38 * "minigzip" tool included in the zlib distribution, not the GNU gzip
39 * program.
40 *
41 * An "imgdiff" patch consists of a header describing the chunk structure
42 * of the file and any encoding parameters needed for the gzipped
43 * chunks, followed by N bsdiff patches, one per chunk.
44 *
45 * For a diff to be generated, the source and target images must have the
46 * same "chunk" structure: that is, the same number of gzipped and normal
47 * chunks in the same order. Android boot and recovery images currently
48 * consist of five chunks: a small normal header, a gzipped kernel, a
49 * small normal section, a gzipped ramdisk, and finally a small normal
50 * footer.
51 *
52 * Caveats: we locate gzipped sections within the source and target
53 * images by searching for the byte sequence 1f8b0800: 1f8b is the gzip
54 * magic number; 08 specifies the "deflate" encoding [the only encoding
55 * supported by the gzip standard]; and 00 is the flags byte. We do not
56 * currently support any extra header fields (which would be indicated by
57 * a nonzero flags byte). We also don't handle the case when that byte
58 * sequence appears spuriously in the file. (Note that it would have to
59 * occur spuriously within a normal chunk to be a problem.)
60 *
61 *
62 * The imgdiff patch header looks like this:
63 *
64 * "IMGDIFF1" (8) [magic number and version]
65 * chunk count (4)
66 * for each chunk:
Doug Zongker6b2bb3d2009-07-20 14:45:29 -070067 * chunk type (4) [CHUNK_{NORMAL, GZIP, DEFLATE, RAW}]
68 * if chunk type == CHUNK_NORMAL:
69 * source start (8)
70 * source len (8)
71 * bsdiff patch offset (8) [from start of patch file]
72 * if chunk type == CHUNK_GZIP: (version 1 only)
73 * source start (8)
74 * source len (8)
75 * bsdiff patch offset (8) [from start of patch file]
Doug Zongker02d444b2009-05-27 18:24:03 -070076 * source expanded len (8) [size of uncompressed source]
77 * target expected len (8) [size of uncompressed target]
78 * gzip level (4)
79 * method (4)
80 * windowBits (4)
81 * memLevel (4)
82 * strategy (4)
83 * gzip header len (4)
84 * gzip header (gzip header len)
85 * gzip footer (8)
Doug Zongker6b2bb3d2009-07-20 14:45:29 -070086 * if chunk type == CHUNK_DEFLATE: (version 2 only)
87 * source start (8)
88 * source len (8)
89 * bsdiff patch offset (8) [from start of patch file]
90 * source expanded len (8) [size of uncompressed source]
91 * target expected len (8) [size of uncompressed target]
92 * gzip level (4)
93 * method (4)
94 * windowBits (4)
95 * memLevel (4)
96 * strategy (4)
97 * if chunk type == RAW: (version 2 only)
98 * target len (4)
99 * data (target len)
Doug Zongker02d444b2009-05-27 18:24:03 -0700100 *
101 * All integers are little-endian. "source start" and "source len"
102 * specify the section of the input image that comprises this chunk,
103 * including the gzip header and footer for gzip chunks. "source
104 * expanded len" is the size of the uncompressed source data. "target
105 * expected len" is the size of the uncompressed data after applying
106 * the bsdiff patch. The next five parameters specify the zlib
107 * parameters to be used when compressing the patched data, and the
108 * next three specify the header and footer to be wrapped around the
109 * compressed data to create the output chunk (so that header contents
110 * like the timestamp are recreated exactly).
111 *
112 * After the header there are 'chunk count' bsdiff patches; the offset
113 * of each from the beginning of the file is specified in the header.
114 */
115
116#include <errno.h>
117#include <stdio.h>
118#include <stdlib.h>
119#include <string.h>
120#include <sys/stat.h>
121#include <unistd.h>
122
123#include "zlib.h"
124#include "imgdiff.h"
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700125#include "utils.h"
Doug Zongker02d444b2009-05-27 18:24:03 -0700126
127typedef struct {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700128 int type; // CHUNK_NORMAL, CHUNK_DEFLATE
Doug Zongker02d444b2009-05-27 18:24:03 -0700129 size_t start; // offset of chunk in original image file
130
131 size_t len;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700132 unsigned char* data; // data to be patched (uncompressed, for deflate chunks)
Doug Zongker02d444b2009-05-27 18:24:03 -0700133
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700134 size_t source_start;
135 size_t source_len;
Doug Zongker02d444b2009-05-27 18:24:03 -0700136
Doug Zongkerd98e0872009-09-25 11:52:00 -0700137 off_t* I; // used by bsdiff
138
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700139 // --- for CHUNK_DEFLATE chunks only: ---
Doug Zongker02d444b2009-05-27 18:24:03 -0700140
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700141 // original (compressed) deflate data
142 size_t deflate_len;
143 unsigned char* deflate_data;
144
145 char* filename; // used for zip entries
Doug Zongker02d444b2009-05-27 18:24:03 -0700146
147 // deflate encoder parameters
148 int level, method, windowBits, memLevel, strategy;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700149
150 size_t source_uncompressed_len;
Doug Zongker02d444b2009-05-27 18:24:03 -0700151} ImageChunk;
152
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700153typedef struct {
154 int data_offset;
155 int deflate_len;
156 int uncomp_len;
157 char* filename;
158} ZipFileEntry;
159
160static int fileentry_compare(const void* a, const void* b) {
161 int ao = ((ZipFileEntry*)a)->data_offset;
162 int bo = ((ZipFileEntry*)b)->data_offset;
163 if (ao < bo) {
164 return -1;
165 } else if (ao > bo) {
166 return 1;
167 } else {
168 return 0;
169 }
170}
171
Doug Zongkerd98e0872009-09-25 11:52:00 -0700172// from bsdiff.c
173int bsdiff(u_char* old, off_t oldsize, off_t** IP, u_char* new, off_t newsize,
174 const char* patch_filename);
175
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700176unsigned char* ReadZip(const char* filename,
177 int* num_chunks, ImageChunk** chunks,
178 int include_pseudo_chunk) {
179 struct stat st;
180 if (stat(filename, &st) != 0) {
181 fprintf(stderr, "failed to stat \"%s\": %s\n", filename, strerror(errno));
182 return NULL;
183 }
184
185 unsigned char* img = malloc(st.st_size);
186 FILE* f = fopen(filename, "rb");
187 if (fread(img, 1, st.st_size, f) != st.st_size) {
188 fprintf(stderr, "failed to read \"%s\" %s\n", filename, strerror(errno));
189 fclose(f);
190 return NULL;
191 }
192 fclose(f);
193
194 // look for the end-of-central-directory record.
195
196 int i;
197 for (i = st.st_size-20; i >= 0 && i > st.st_size - 65600; --i) {
198 if (img[i] == 0x50 && img[i+1] == 0x4b &&
199 img[i+2] == 0x05 && img[i+3] == 0x06) {
200 break;
201 }
202 }
203 // double-check: this archive consists of a single "disk"
204 if (!(img[i+4] == 0 && img[i+5] == 0 && img[i+6] == 0 && img[i+7] == 0)) {
205 fprintf(stderr, "can't process multi-disk archive\n");
206 return NULL;
207 }
208
209 int cdcount = Read2(img+i+8);
210 int cdoffset = Read4(img+i+16);
211
212 ZipFileEntry* temp_entries = malloc(cdcount * sizeof(ZipFileEntry));
213 int entrycount = 0;
214
215 unsigned char* cd = img+cdoffset;
216 for (i = 0; i < cdcount; ++i) {
217 if (!(cd[0] == 0x50 && cd[1] == 0x4b && cd[2] == 0x01 && cd[3] == 0x02)) {
218 fprintf(stderr, "bad central directory entry %d\n", i);
219 return NULL;
220 }
221
222 int clen = Read4(cd+20); // compressed len
223 int ulen = Read4(cd+24); // uncompressed len
224 int nlen = Read2(cd+28); // filename len
225 int xlen = Read2(cd+30); // extra field len
226 int mlen = Read2(cd+32); // file comment len
227 int hoffset = Read4(cd+42); // local header offset
228
229 char* filename = malloc(nlen+1);
230 memcpy(filename, cd+46, nlen);
231 filename[nlen] = '\0';
232
233 int method = Read2(cd+10);
234
235 cd += 46 + nlen + xlen + mlen;
236
237 if (method != 8) { // 8 == deflate
238 free(filename);
239 continue;
240 }
241
242 unsigned char* lh = img + hoffset;
243
244 if (!(lh[0] == 0x50 && lh[1] == 0x4b && lh[2] == 0x03 && lh[3] == 0x04)) {
245 fprintf(stderr, "bad local file header entry %d\n", i);
246 return NULL;
247 }
248
249 if (Read2(lh+26) != nlen || memcmp(lh+30, filename, nlen) != 0) {
250 fprintf(stderr, "central dir filename doesn't match local header\n");
251 return NULL;
252 }
253
254 xlen = Read2(lh+28); // extra field len; might be different from CD entry?
255
256 temp_entries[entrycount].data_offset = hoffset+30+nlen+xlen;
257 temp_entries[entrycount].deflate_len = clen;
258 temp_entries[entrycount].uncomp_len = ulen;
259 temp_entries[entrycount].filename = filename;
260 ++entrycount;
261 }
262
263 qsort(temp_entries, entrycount, sizeof(ZipFileEntry), fileentry_compare);
264
265#if 0
266 printf("found %d deflated entries\n", entrycount);
267 for (i = 0; i < entrycount; ++i) {
268 printf("off %10d len %10d unlen %10d %p %s\n",
269 temp_entries[i].data_offset,
270 temp_entries[i].deflate_len,
271 temp_entries[i].uncomp_len,
272 temp_entries[i].filename,
273 temp_entries[i].filename);
274 }
275#endif
276
277 *num_chunks = 0;
278 *chunks = malloc((entrycount*2+2) * sizeof(ImageChunk));
279 ImageChunk* curr = *chunks;
280
281 if (include_pseudo_chunk) {
282 curr->type = CHUNK_NORMAL;
283 curr->start = 0;
284 curr->len = st.st_size;
285 curr->data = img;
286 curr->filename = NULL;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700287 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700288 ++curr;
289 ++*num_chunks;
290 }
291
292 int pos = 0;
293 int nextentry = 0;
294
295 while (pos < st.st_size) {
296 if (nextentry < entrycount && pos == temp_entries[nextentry].data_offset) {
297 curr->type = CHUNK_DEFLATE;
298 curr->start = pos;
299 curr->deflate_len = temp_entries[nextentry].deflate_len;
300 curr->deflate_data = img + pos;
301 curr->filename = temp_entries[nextentry].filename;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700302 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700303
304 curr->len = temp_entries[nextentry].uncomp_len;
305 curr->data = malloc(curr->len);
306
307 z_stream strm;
308 strm.zalloc = Z_NULL;
309 strm.zfree = Z_NULL;
310 strm.opaque = Z_NULL;
311 strm.avail_in = curr->deflate_len;
312 strm.next_in = curr->deflate_data;
313
314 // -15 means we are decoding a 'raw' deflate stream; zlib will
315 // not expect zlib headers.
316 int ret = inflateInit2(&strm, -15);
317
318 strm.avail_out = curr->len;
319 strm.next_out = curr->data;
320 ret = inflate(&strm, Z_NO_FLUSH);
321 if (ret != Z_STREAM_END) {
322 fprintf(stderr, "failed to inflate \"%s\"; %d\n", curr->filename, ret);
323 return NULL;
324 }
325
326 inflateEnd(&strm);
327
328 pos += curr->deflate_len;
329 ++nextentry;
330 ++*num_chunks;
331 ++curr;
332 continue;
333 }
334
335 // use a normal chunk to take all the data up to the start of the
336 // next deflate section.
337
338 curr->type = CHUNK_NORMAL;
339 curr->start = pos;
340 if (nextentry < entrycount) {
341 curr->len = temp_entries[nextentry].data_offset - pos;
342 } else {
343 curr->len = st.st_size - pos;
344 }
345 curr->data = img + pos;
346 curr->filename = NULL;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700347 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700348 pos += curr->len;
349
350 ++*num_chunks;
351 ++curr;
352 }
353
354 free(temp_entries);
355 return img;
356}
357
Doug Zongker02d444b2009-05-27 18:24:03 -0700358/*
359 * Read the given file and break it up into chunks, putting the number
360 * of chunks and their info in *num_chunks and **chunks,
361 * respectively. Returns a malloc'd block of memory containing the
362 * contents of the file; various pointers in the output chunk array
363 * will point into this block of memory. The caller should free the
364 * return value when done with all the chunks. Returns NULL on
365 * failure.
366 */
367unsigned char* ReadImage(const char* filename,
368 int* num_chunks, ImageChunk** chunks) {
369 struct stat st;
370 if (stat(filename, &st) != 0) {
371 fprintf(stderr, "failed to stat \"%s\": %s\n", filename, strerror(errno));
372 return NULL;
373 }
374
375 unsigned char* img = malloc(st.st_size + 4);
376 FILE* f = fopen(filename, "rb");
377 if (fread(img, 1, st.st_size, f) != st.st_size) {
378 fprintf(stderr, "failed to read \"%s\" %s\n", filename, strerror(errno));
379 fclose(f);
380 return NULL;
381 }
382 fclose(f);
383
384 // append 4 zero bytes to the data so we can always search for the
385 // four-byte string 1f8b0800 starting at any point in the actual
386 // file data, without special-casing the end of the data.
387 memset(img+st.st_size, 0, 4);
388
389 size_t pos = 0;
390
391 *num_chunks = 0;
392 *chunks = NULL;
393
394 while (pos < st.st_size) {
395 unsigned char* p = img+pos;
396
Doug Zongker02d444b2009-05-27 18:24:03 -0700397 if (st.st_size - pos >= 4 &&
398 p[0] == 0x1f && p[1] == 0x8b &&
399 p[2] == 0x08 && // deflate compression
400 p[3] == 0x00) { // no header flags
401 // 'pos' is the offset of the start of a gzip chunk.
402
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700403 *num_chunks += 3;
404 *chunks = realloc(*chunks, *num_chunks * sizeof(ImageChunk));
405 ImageChunk* curr = *chunks + (*num_chunks-3);
406
407 // create a normal chunk for the header.
408 curr->start = pos;
409 curr->type = CHUNK_NORMAL;
410 curr->len = GZIP_HEADER_LEN;
411 curr->data = p;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700412 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700413
414 pos += curr->len;
415 p += curr->len;
416 ++curr;
417
418 curr->type = CHUNK_DEFLATE;
419 curr->filename = NULL;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700420 curr->I = NULL;
Doug Zongker02d444b2009-05-27 18:24:03 -0700421
422 // We must decompress this chunk in order to discover where it
423 // ends, and so we can put the uncompressed data and its length
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700424 // into curr->data and curr->len.
Doug Zongker02d444b2009-05-27 18:24:03 -0700425
426 size_t allocated = 32768;
427 curr->len = 0;
428 curr->data = malloc(allocated);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700429 curr->start = pos;
430 curr->deflate_data = p;
Doug Zongker02d444b2009-05-27 18:24:03 -0700431
432 z_stream strm;
433 strm.zalloc = Z_NULL;
434 strm.zfree = Z_NULL;
435 strm.opaque = Z_NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700436 strm.avail_in = st.st_size - pos;
437 strm.next_in = p;
Doug Zongker02d444b2009-05-27 18:24:03 -0700438
439 // -15 means we are decoding a 'raw' deflate stream; zlib will
440 // not expect zlib headers.
441 int ret = inflateInit2(&strm, -15);
442
443 do {
444 strm.avail_out = allocated - curr->len;
445 strm.next_out = curr->data + curr->len;
446 ret = inflate(&strm, Z_NO_FLUSH);
447 curr->len = allocated - strm.avail_out;
448 if (strm.avail_out == 0) {
449 allocated *= 2;
450 curr->data = realloc(curr->data, allocated);
451 }
452 } while (ret != Z_STREAM_END);
453
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700454 curr->deflate_len = st.st_size - strm.avail_in - pos;
Doug Zongker02d444b2009-05-27 18:24:03 -0700455 inflateEnd(&strm);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700456 pos += curr->deflate_len;
457 p += curr->deflate_len;
458 ++curr;
Doug Zongker02d444b2009-05-27 18:24:03 -0700459
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700460 // create a normal chunk for the footer
461
462 curr->type = CHUNK_NORMAL;
463 curr->start = pos;
464 curr->len = GZIP_FOOTER_LEN;
465 curr->data = img+pos;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700466 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700467
468 pos += curr->len;
469 p += curr->len;
470 ++curr;
Doug Zongker02d444b2009-05-27 18:24:03 -0700471
472 // The footer (that we just skipped over) contains the size of
473 // the uncompressed data. Double-check to make sure that it
474 // matches the size of the data we got when we actually did
475 // the decompression.
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700476 size_t footer_size = Read4(p-4);
477 if (footer_size != curr[-2].len) {
Doug Zongker02d444b2009-05-27 18:24:03 -0700478 fprintf(stderr, "Error: footer size %d != decompressed size %d\n",
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700479 footer_size, curr[-2].len);
Doug Zongker02d444b2009-05-27 18:24:03 -0700480 free(img);
481 return NULL;
482 }
483 } else {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700484 // Reallocate the list for every chunk; we expect the number of
485 // chunks to be small (5 for typical boot and recovery images).
486 ++*num_chunks;
487 *chunks = realloc(*chunks, *num_chunks * sizeof(ImageChunk));
488 ImageChunk* curr = *chunks + (*num_chunks-1);
489 curr->start = pos;
Doug Zongkerd98e0872009-09-25 11:52:00 -0700490 curr->I = NULL;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700491
Doug Zongker02d444b2009-05-27 18:24:03 -0700492 // 'pos' is not the offset of the start of a gzip chunk, so scan
493 // forward until we find a gzip header.
494 curr->type = CHUNK_NORMAL;
495 curr->data = p;
496
497 for (curr->len = 0; curr->len < (st.st_size - pos); ++curr->len) {
498 if (p[curr->len] == 0x1f &&
499 p[curr->len+1] == 0x8b &&
500 p[curr->len+2] == 0x08 &&
501 p[curr->len+3] == 0x00) {
502 break;
503 }
504 }
505 pos += curr->len;
506 }
507 }
508
509 return img;
510}
511
512#define BUFFER_SIZE 32768
513
514/*
515 * Takes the uncompressed data stored in the chunk, compresses it
516 * using the zlib parameters stored in the chunk, and checks that it
517 * matches exactly the compressed data we started with (also stored in
518 * the chunk). Return 0 on success.
519 */
520int TryReconstruction(ImageChunk* chunk, unsigned char* out) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700521 size_t p = 0;
522
523#if 0
524 fprintf(stderr, "trying %d %d %d %d %d\n",
525 chunk->level, chunk->method, chunk->windowBits,
526 chunk->memLevel, chunk->strategy);
527#endif
Doug Zongker02d444b2009-05-27 18:24:03 -0700528
529 z_stream strm;
530 strm.zalloc = Z_NULL;
531 strm.zfree = Z_NULL;
532 strm.opaque = Z_NULL;
533 strm.avail_in = chunk->len;
534 strm.next_in = chunk->data;
535 int ret;
536 ret = deflateInit2(&strm, chunk->level, chunk->method, chunk->windowBits,
537 chunk->memLevel, chunk->strategy);
538 do {
539 strm.avail_out = BUFFER_SIZE;
540 strm.next_out = out;
541 ret = deflate(&strm, Z_FINISH);
542 size_t have = BUFFER_SIZE - strm.avail_out;
543
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700544 if (memcmp(out, chunk->deflate_data+p, have) != 0) {
Doug Zongker02d444b2009-05-27 18:24:03 -0700545 // mismatch; data isn't the same.
546 deflateEnd(&strm);
547 return -1;
548 }
549 p += have;
550 } while (ret != Z_STREAM_END);
551 deflateEnd(&strm);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700552 if (p != chunk->deflate_len) {
Doug Zongker02d444b2009-05-27 18:24:03 -0700553 // mismatch; ran out of data before we should have.
554 return -1;
555 }
556 return 0;
557}
558
559/*
560 * Verify that we can reproduce exactly the same compressed data that
561 * we started with. Sets the level, method, windowBits, memLevel, and
562 * strategy fields in the chunk to the encoding parameters needed to
563 * produce the right output. Returns 0 on success.
564 */
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700565int ReconstructDeflateChunk(ImageChunk* chunk) {
566 if (chunk->type != CHUNK_DEFLATE) {
567 fprintf(stderr, "attempt to reconstruct non-deflate chunk\n");
Doug Zongker02d444b2009-05-27 18:24:03 -0700568 return -1;
569 }
570
571 size_t p = 0;
572 unsigned char* out = malloc(BUFFER_SIZE);
573
574 // We only check two combinations of encoder parameters: level 6
575 // (the default) and level 9 (the maximum).
576 for (chunk->level = 6; chunk->level <= 9; chunk->level += 3) {
577 chunk->windowBits = -15; // 32kb window; negative to indicate a raw stream.
578 chunk->memLevel = 8; // the default value.
579 chunk->method = Z_DEFLATED;
580 chunk->strategy = Z_DEFAULT_STRATEGY;
581
582 if (TryReconstruction(chunk, out) == 0) {
583 free(out);
584 return 0;
585 }
586 }
587
588 free(out);
589 return -1;
590}
591
Doug Zongker02d444b2009-05-27 18:24:03 -0700592/*
593 * Given source and target chunks, compute a bsdiff patch between them
594 * by running bsdiff in a subprocess. Return the patch data, placing
595 * its length in *size. Return NULL on failure. We expect the bsdiff
596 * program to be in the path.
597 */
598unsigned char* MakePatch(ImageChunk* src, ImageChunk* tgt, size_t* size) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700599 if (tgt->type == CHUNK_NORMAL) {
600 if (tgt->len <= 160) {
601 tgt->type = CHUNK_RAW;
602 *size = tgt->len;
603 return tgt->data;
604 }
605 }
606
Doug Zongker02d444b2009-05-27 18:24:03 -0700607 char ptemp[] = "/tmp/imgdiff-patch-XXXXXX";
Doug Zongker02d444b2009-05-27 18:24:03 -0700608 mkstemp(ptemp);
609
Doug Zongkerd98e0872009-09-25 11:52:00 -0700610 int r = bsdiff(src->data, src->len, &(src->I), tgt->data, tgt->len, ptemp);
611 if (r != 0) {
612 fprintf(stderr, "bsdiff() failed: %d\n", r);
Doug Zongker02d444b2009-05-27 18:24:03 -0700613 return NULL;
614 }
615
616 struct stat st;
617 if (stat(ptemp, &st) != 0) {
618 fprintf(stderr, "failed to stat patch file %s: %s\n",
619 ptemp, strerror(errno));
620 return NULL;
621 }
622
623 unsigned char* data = malloc(st.st_size);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700624
625 if (tgt->type == CHUNK_NORMAL && tgt->len <= st.st_size) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700626 unlink(ptemp);
627
628 tgt->type = CHUNK_RAW;
629 *size = tgt->len;
630 return tgt->data;
631 }
632
Doug Zongker02d444b2009-05-27 18:24:03 -0700633 *size = st.st_size;
634
Doug Zongkerd98e0872009-09-25 11:52:00 -0700635 FILE* f = fopen(ptemp, "rb");
Doug Zongker02d444b2009-05-27 18:24:03 -0700636 if (f == NULL) {
637 fprintf(stderr, "failed to open patch %s: %s\n", ptemp, strerror(errno));
638 return NULL;
639 }
640 if (fread(data, 1, st.st_size, f) != st.st_size) {
641 fprintf(stderr, "failed to read patch %s: %s\n", ptemp, strerror(errno));
642 return NULL;
643 }
644 fclose(f);
645
Doug Zongker02d444b2009-05-27 18:24:03 -0700646 unlink(ptemp);
647
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700648 tgt->source_start = src->start;
649 switch (tgt->type) {
650 case CHUNK_NORMAL:
651 tgt->source_len = src->len;
652 break;
653 case CHUNK_DEFLATE:
654 tgt->source_len = src->deflate_len;
655 tgt->source_uncompressed_len = src->len;
656 break;
657 }
658
Doug Zongker02d444b2009-05-27 18:24:03 -0700659 return data;
660}
661
662/*
663 * Cause a gzip chunk to be treated as a normal chunk (ie, as a blob
664 * of uninterpreted data). The resulting patch will likely be about
665 * as big as the target file, but it lets us handle the case of images
666 * where some gzip chunks are reconstructible but others aren't (by
667 * treating the ones that aren't as normal chunks).
668 */
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700669void ChangeDeflateChunkToNormal(ImageChunk* ch) {
670 if (ch->type != CHUNK_DEFLATE) return;
Doug Zongker02d444b2009-05-27 18:24:03 -0700671 ch->type = CHUNK_NORMAL;
672 free(ch->data);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700673 ch->data = ch->deflate_data;
674 ch->len = ch->deflate_len;
Doug Zongker02d444b2009-05-27 18:24:03 -0700675}
676
Doug Zongker3b724362009-07-15 17:54:30 -0700677/*
678 * Return true if the data in the chunk is identical (including the
679 * compressed representation, for gzip chunks).
680 */
681int AreChunksEqual(ImageChunk* a, ImageChunk* b) {
682 if (a->type != b->type) return 0;
683
684 switch (a->type) {
685 case CHUNK_NORMAL:
686 return a->len == b->len && memcmp(a->data, b->data, a->len) == 0;
687
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700688 case CHUNK_DEFLATE:
689 return a->deflate_len == b->deflate_len &&
690 memcmp(a->deflate_data, b->deflate_data, a->deflate_len) == 0;
Doug Zongker3b724362009-07-15 17:54:30 -0700691
692 default:
693 fprintf(stderr, "unknown chunk type %d\n", a->type);
694 return 0;
695 }
696}
697
698/*
699 * Look for runs of adjacent normal chunks and compress them down into
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700700 * a single chunk. (Such runs can be produced when deflate chunks are
Doug Zongker3b724362009-07-15 17:54:30 -0700701 * changed to normal chunks.)
702 */
703void MergeAdjacentNormalChunks(ImageChunk* chunks, int* num_chunks) {
704 int out = 0;
705 int in_start = 0, in_end;
706 while (in_start < *num_chunks) {
707 if (chunks[in_start].type != CHUNK_NORMAL) {
708 in_end = in_start+1;
709 } else {
710 // in_start is a normal chunk. Look for a run of normal chunks
711 // that constitute a solid block of data (ie, each chunk begins
712 // where the previous one ended).
713 for (in_end = in_start+1;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700714 in_end < *num_chunks && chunks[in_end].type == CHUNK_NORMAL &&
Doug Zongker3b724362009-07-15 17:54:30 -0700715 (chunks[in_end].start ==
716 chunks[in_end-1].start + chunks[in_end-1].len &&
717 chunks[in_end].data ==
718 chunks[in_end-1].data + chunks[in_end-1].len);
719 ++in_end);
720 }
721
722 if (in_end == in_start+1) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700723#if 0
724 printf("chunk %d is now %d\n", in_start, out);
725#endif
Doug Zongker3b724362009-07-15 17:54:30 -0700726 if (out != in_start) {
727 memcpy(chunks+out, chunks+in_start, sizeof(ImageChunk));
728 }
729 } else {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700730#if 0
731 printf("collapse normal chunks %d-%d into %d\n", in_start, in_end-1, out);
732#endif
Doug Zongker3b724362009-07-15 17:54:30 -0700733
734 // Merge chunks [in_start, in_end-1] into one chunk. Since the
735 // data member of each chunk is just a pointer into an in-memory
736 // copy of the file, this can be done without recopying (the
737 // output chunk has the first chunk's start location and data
738 // pointer, and length equal to the sum of the input chunk
739 // lengths).
740 chunks[out].type = CHUNK_NORMAL;
741 chunks[out].start = chunks[in_start].start;
742 chunks[out].data = chunks[in_start].data;
743 chunks[out].len = chunks[in_end-1].len +
744 (chunks[in_end-1].start - chunks[in_start].start);
745 }
746
747 ++out;
748 in_start = in_end;
749 }
750 *num_chunks = out;
751}
752
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700753ImageChunk* FindChunkByName(const char* name,
754 ImageChunk* chunks, int num_chunks) {
755 int i;
756 for (i = 0; i < num_chunks; ++i) {
757 if (chunks[i].type == CHUNK_DEFLATE && chunks[i].filename &&
758 strcmp(name, chunks[i].filename) == 0) {
759 return chunks+i;
760 }
761 }
762 return NULL;
763}
764
Doug Zongkerd7d7ab02009-09-02 11:10:51 -0700765void DumpChunks(ImageChunk* chunks, int num_chunks) {
766 int i;
767 for (i = 0; i < num_chunks; ++i) {
768 printf("chunk %d: type %d start %d len %d\n",
769 i, chunks[i].type, chunks[i].start, chunks[i].len);
770 }
771}
772
Doug Zongker02d444b2009-05-27 18:24:03 -0700773int main(int argc, char** argv) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700774 if (argc != 4 && argc != 5) {
775 usage:
776 fprintf(stderr, "usage: %s [-z] <src-img> <tgt-img> <patch-file>\n",
777 argv[0]);
Doug Zongker02d444b2009-05-27 18:24:03 -0700778 return 2;
779 }
780
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700781 int zip_mode = 0;
782
783 if (strcmp(argv[1], "-z") == 0) {
784 zip_mode = 1;
785 --argc;
786 ++argv;
787 }
788
789
Doug Zongker02d444b2009-05-27 18:24:03 -0700790 int num_src_chunks;
791 ImageChunk* src_chunks;
Doug Zongker02d444b2009-05-27 18:24:03 -0700792 int num_tgt_chunks;
793 ImageChunk* tgt_chunks;
Doug Zongker02d444b2009-05-27 18:24:03 -0700794 int i;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700795
796 if (zip_mode) {
797 if (ReadZip(argv[1], &num_src_chunks, &src_chunks, 1) == NULL) {
798 fprintf(stderr, "failed to break apart source zip file\n");
Doug Zongker02d444b2009-05-27 18:24:03 -0700799 return 1;
800 }
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700801 if (ReadZip(argv[2], &num_tgt_chunks, &tgt_chunks, 0) == NULL) {
802 fprintf(stderr, "failed to break apart target zip file\n");
803 return 1;
804 }
805 } else {
806 if (ReadImage(argv[1], &num_src_chunks, &src_chunks) == NULL) {
807 fprintf(stderr, "failed to break apart source image\n");
808 return 1;
809 }
810 if (ReadImage(argv[2], &num_tgt_chunks, &tgt_chunks) == NULL) {
811 fprintf(stderr, "failed to break apart target image\n");
812 return 1;
813 }
814
815 // Verify that the source and target images have the same chunk
816 // structure (ie, the same sequence of deflate and normal chunks).
817
Doug Zongkerd7d7ab02009-09-02 11:10:51 -0700818 if (!zip_mode) {
819 // Merge the gzip header and footer in with any adjacent
820 // normal chunks.
821 MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks);
822 MergeAdjacentNormalChunks(src_chunks, &num_src_chunks);
823 }
824
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700825 if (num_src_chunks != num_tgt_chunks) {
826 fprintf(stderr, "source and target don't have same number of chunks!\n");
Doug Zongkerd7d7ab02009-09-02 11:10:51 -0700827 printf("source chunks:\n");
828 DumpChunks(src_chunks, num_src_chunks);
829 printf("target chunks:\n");
830 DumpChunks(tgt_chunks, num_tgt_chunks);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700831 return 1;
832 }
833 for (i = 0; i < num_src_chunks; ++i) {
834 if (src_chunks[i].type != tgt_chunks[i].type) {
835 fprintf(stderr, "source and target don't have same chunk "
836 "structure! (chunk %d)\n", i);
Doug Zongkerd7d7ab02009-09-02 11:10:51 -0700837 printf("source chunks:\n");
838 DumpChunks(src_chunks, num_src_chunks);
839 printf("target chunks:\n");
840 DumpChunks(tgt_chunks, num_tgt_chunks);
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700841 return 1;
842 }
843 }
Doug Zongker02d444b2009-05-27 18:24:03 -0700844 }
845
Doug Zongker02d444b2009-05-27 18:24:03 -0700846 for (i = 0; i < num_tgt_chunks; ++i) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700847 if (tgt_chunks[i].type == CHUNK_DEFLATE) {
Doug Zongker3b724362009-07-15 17:54:30 -0700848 // Confirm that given the uncompressed chunk data in the target, we
849 // can recompress it and get exactly the same bits as are in the
850 // input target image. If this fails, treat the chunk as a normal
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700851 // non-deflated chunk.
852 if (ReconstructDeflateChunk(tgt_chunks+i) < 0) {
853 printf("failed to reconstruct target deflate chunk %d [%s]; "
854 "treating as normal\n", i, tgt_chunks[i].filename);
855 ChangeDeflateChunkToNormal(tgt_chunks+i);
856 if (zip_mode) {
857 ImageChunk* src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks);
858 if (src) {
859 ChangeDeflateChunkToNormal(src);
860 }
861 } else {
862 ChangeDeflateChunkToNormal(src_chunks+i);
863 }
Doug Zongker3b724362009-07-15 17:54:30 -0700864 continue;
Doug Zongker02d444b2009-05-27 18:24:03 -0700865 }
Doug Zongker3b724362009-07-15 17:54:30 -0700866
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700867 // If two deflate chunks are identical (eg, the kernel has not
Doug Zongker3b724362009-07-15 17:54:30 -0700868 // changed between two builds), treat them as normal chunks.
869 // This makes applypatch much faster -- it can apply a trivial
870 // patch to the compressed data, rather than uncompressing and
871 // recompressing to apply the trivial patch to the uncompressed
872 // data.
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700873 ImageChunk* src;
874 if (zip_mode) {
875 src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks);
876 } else {
877 src = src_chunks+i;
878 }
879
880 if (src == NULL || AreChunksEqual(tgt_chunks+i, src)) {
881 ChangeDeflateChunkToNormal(tgt_chunks+i);
882 if (src) {
883 ChangeDeflateChunkToNormal(src);
884 }
Doug Zongker3b724362009-07-15 17:54:30 -0700885 }
Doug Zongker02d444b2009-05-27 18:24:03 -0700886 }
887 }
888
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700889 // Merging neighboring normal chunks.
890 if (zip_mode) {
891 // For zips, we only need to do this to the target: deflated
892 // chunks are matched via filename, and normal chunks are patched
893 // using the entire source file as the source.
894 MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks);
895 } else {
896 // For images, we need to maintain the parallel structure of the
897 // chunk lists, so do the merging in both the source and target
898 // lists.
899 MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks);
900 MergeAdjacentNormalChunks(src_chunks, &num_src_chunks);
901 if (num_src_chunks != num_tgt_chunks) {
902 // This shouldn't happen.
903 fprintf(stderr, "merging normal chunks went awry\n");
904 return 1;
905 }
Doug Zongker3b724362009-07-15 17:54:30 -0700906 }
907
Doug Zongker02d444b2009-05-27 18:24:03 -0700908 // Compute bsdiff patches for each chunk's data (the uncompressed
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700909 // data, in the case of deflate chunks).
Doug Zongker02d444b2009-05-27 18:24:03 -0700910
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700911 printf("Construct patches for %d chunks...\n", num_tgt_chunks);
912 unsigned char** patch_data = malloc(num_tgt_chunks * sizeof(unsigned char*));
913 size_t* patch_size = malloc(num_tgt_chunks * sizeof(size_t));
914 for (i = 0; i < num_tgt_chunks; ++i) {
915 if (zip_mode) {
916 ImageChunk* src;
917 if (tgt_chunks[i].type == CHUNK_DEFLATE &&
918 (src = FindChunkByName(tgt_chunks[i].filename, src_chunks,
919 num_src_chunks))) {
920 patch_data[i] = MakePatch(src, tgt_chunks+i, patch_size+i);
921 } else {
922 patch_data[i] = MakePatch(src_chunks, tgt_chunks+i, patch_size+i);
923 }
924 } else {
925 patch_data[i] = MakePatch(src_chunks+i, tgt_chunks+i, patch_size+i);
926 }
927 printf("patch %3d is %d bytes (of %d)\n",
928 i, patch_size[i], tgt_chunks[i].source_len);
Doug Zongker02d444b2009-05-27 18:24:03 -0700929 }
930
931 // Figure out how big the imgdiff file header is going to be, so
932 // that we can correctly compute the offset of each bsdiff patch
933 // within the file.
934
935 size_t total_header_size = 12;
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700936 for (i = 0; i < num_tgt_chunks; ++i) {
937 total_header_size += 4;
938 switch (tgt_chunks[i].type) {
939 case CHUNK_NORMAL:
940 total_header_size += 8*3;
941 break;
942 case CHUNK_DEFLATE:
943 total_header_size += 8*5 + 4*5;
944 break;
945 case CHUNK_RAW:
946 total_header_size += 4 + patch_size[i];
947 break;
Doug Zongker02d444b2009-05-27 18:24:03 -0700948 }
949 }
950
951 size_t offset = total_header_size;
952
953 FILE* f = fopen(argv[3], "wb");
954
955 // Write out the headers.
956
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700957 fwrite("IMGDIFF2", 1, 8, f);
958 Write4(num_tgt_chunks, f);
Doug Zongker02d444b2009-05-27 18:24:03 -0700959 for (i = 0; i < num_tgt_chunks; ++i) {
960 Write4(tgt_chunks[i].type, f);
Doug Zongker02d444b2009-05-27 18:24:03 -0700961
Doug Zongker6b2bb3d2009-07-20 14:45:29 -0700962 switch (tgt_chunks[i].type) {
963 case CHUNK_NORMAL:
964 printf("chunk %3d: normal (%10d, %10d) %10d\n", i,
965 tgt_chunks[i].start, tgt_chunks[i].len, patch_size[i]);
966 Write8(tgt_chunks[i].source_start, f);
967 Write8(tgt_chunks[i].source_len, f);
968 Write8(offset, f);
969 offset += patch_size[i];
970 break;
971
972 case CHUNK_DEFLATE:
973 printf("chunk %3d: deflate (%10d, %10d) %10d %s\n", i,
974 tgt_chunks[i].start, tgt_chunks[i].deflate_len, patch_size[i],
975 tgt_chunks[i].filename);
976 Write8(tgt_chunks[i].source_start, f);
977 Write8(tgt_chunks[i].source_len, f);
978 Write8(offset, f);
979 Write8(tgt_chunks[i].source_uncompressed_len, f);
980 Write8(tgt_chunks[i].len, f);
981 Write4(tgt_chunks[i].level, f);
982 Write4(tgt_chunks[i].method, f);
983 Write4(tgt_chunks[i].windowBits, f);
984 Write4(tgt_chunks[i].memLevel, f);
985 Write4(tgt_chunks[i].strategy, f);
986 offset += patch_size[i];
987 break;
988
989 case CHUNK_RAW:
990 printf("chunk %3d: raw (%10d, %10d)\n", i,
991 tgt_chunks[i].start, tgt_chunks[i].len);
992 Write4(patch_size[i], f);
993 fwrite(patch_data[i], 1, patch_size[i], f);
994 break;
Doug Zongker02d444b2009-05-27 18:24:03 -0700995 }
Doug Zongker02d444b2009-05-27 18:24:03 -0700996 }
997
998 // Append each chunk's bsdiff patch, in order.
999
1000 for (i = 0; i < num_tgt_chunks; ++i) {
Doug Zongker6b2bb3d2009-07-20 14:45:29 -07001001 if (tgt_chunks[i].type != CHUNK_RAW) {
1002 fwrite(patch_data[i], 1, patch_size[i], f);
1003 }
Doug Zongker02d444b2009-05-27 18:24:03 -07001004 }
1005
1006 fclose(f);
1007
1008 return 0;
1009}