From ef85ea6086107537d07720283ca9b1af01de247c Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Fri, 8 May 2009 13:46:25 -0700 Subject: [PATCH] allow separate source and target files for applypatch Allow applypatch to use different filenames for the source and target. (Using the same filename is still fine; in fact the target filename can be specified as "-" to mean "same as the source file.) This will allow us to still use diffs in the case of files being renamed, and will allow us to use diffs for the boot and recovery images. --- tools/applypatch/applypatch.c | 71 +++++++++----- tools/applypatch/applypatch.sh | 115 ++++++++++++++++++----- tools/releasetools/ota_from_target_files | 2 +- 3 files changed, 141 insertions(+), 47 deletions(-) diff --git a/tools/applypatch/applypatch.c b/tools/applypatch/applypatch.c index 9954869c7e..23b41d772f 100644 --- a/tools/applypatch/applypatch.c +++ b/tools/applypatch/applypatch.c @@ -43,6 +43,7 @@ int LoadFileContents(const char* filename, FileContents* file) { if (f == NULL) { fprintf(stderr, "failed to open \"%s\": %s\n", filename, strerror(errno)); free(file->data); + file->data = NULL; return -1; } @@ -51,6 +52,7 @@ int LoadFileContents(const char* filename, FileContents* file) { fprintf(stderr, "short read of \"%s\" (%d bytes of %d)\n", filename, bytes_read, file->size); free(file->data); + file->data = NULL; return -1; } fclose(f); @@ -226,14 +228,16 @@ size_t FreeSpaceForFile(const char* filename) { // replacement for it) and idempotent (it's okay to run this program // multiple times). // -// - if the sha1 hash of is , does nothing and exits +// - if the sha1 hash of is , does nothing and exits // successfully. // -// - otherwise, if the sha1 hash of is , applies the -// bsdiff to to produce a new file (the type of patch +// - otherwise, if the sha1 hash of is , applies the +// bsdiff to to produce a new file (the type of patch // is automatically detected from the file header). If that new -// file has sha1 hash , moves it to replace , and -// exits successfully. +// file has sha1 hash , moves it to replace , and +// exits successfully. Note that if and are +// not the same, is NOT deleted on success. +// may be the string "-" to mean "the same as src-file". // // - otherwise, or if any error is encountered, exits with non-zero // status. @@ -241,7 +245,7 @@ size_t FreeSpaceForFile(const char* filename) { int main(int argc, char** argv) { if (argc < 2) { usage: - fprintf(stderr, "usage: %s [: ...]\n" + fprintf(stderr, "usage: %s [: ...]\n" " or %s -c [ ...]\n" " or %s -s \n" " or %s -l\n", @@ -273,26 +277,31 @@ int main(int argc, char** argv) { uint8_t target_sha1[SHA_DIGEST_SIZE]; const char* source_filename = argv[1]; + const char* target_filename = argv[2]; + if (target_filename[0] == '-' && + target_filename[1] == '\0') { + target_filename = source_filename; + } - // assume that source_filename (eg "/system/app/Foo.apk") is located + // assume that target_filename (eg "/system/app/Foo.apk") is located // on the same filesystem as its top-level directory ("/system"). // We need something that exists for calling statfs(). - char* source_fs = strdup(argv[1]); - char* slash = strchr(source_fs+1, '/'); + char* target_fs = strdup(target_filename); + char* slash = strchr(target_fs+1, '/'); if (slash != NULL) { *slash = '\0'; } - if (ParseSha1(argv[2], target_sha1) != 0) { - fprintf(stderr, "failed to parse tgt-sha1 \"%s\"\n", argv[2]); + if (ParseSha1(argv[3], target_sha1) != 0) { + fprintf(stderr, "failed to parse tgt-sha1 \"%s\"\n", argv[3]); return 1; } - unsigned long target_size = strtoul(argv[3], NULL, 0); + unsigned long target_size = strtoul(argv[4], NULL, 0); int num_patches; Patch* patches; - if (ParseShaArgs(argc-4, argv+4, &patches, &num_patches) < 0) { return 1; } + if (ParseShaArgs(argc-5, argv+5, &patches, &num_patches) < 0) { return 1; } FileContents copy_file; FileContents source_file; @@ -300,15 +309,27 @@ int main(int argc, char** argv) { const char* copy_patch_filename = NULL; int made_copy = 0; - if (LoadFileContents(source_filename, &source_file) == 0) { + // We try to load the target file into the source_file object. + if (LoadFileContents(target_filename, &source_file) == 0) { if (memcmp(source_file.sha1, target_sha1, SHA_DIGEST_SIZE) == 0) { // The early-exit case: the patch was already applied, this file // has the desired hash, nothing for us to do. fprintf(stderr, "\"%s\" is already target; no patch needed\n", - source_filename); + target_filename); return 0; } + } + if (source_file.data == NULL || + (target_filename != source_filename && + strcmp(target_filename, source_filename) != 0)) { + // Need to load the source file: either we failed to load the + // target file, or we did but it's different from the source file. + free(source_file.data); + LoadFileContents(source_filename, &source_file); + } + + if (source_file.data != NULL) { const Patch* to_use = FindMatchingPatch(source_file.sha1, patches, num_patches); if (to_use != NULL) { @@ -340,7 +361,7 @@ int main(int argc, char** argv) { } // Is there enough room in the target filesystem to hold the patched file? - size_t free_space = FreeSpaceForFile(source_fs); + size_t free_space = FreeSpaceForFile(target_fs); int enough_space = free_space > (target_size * 3 / 2); // 50% margin of error printf("target %ld bytes; free space %ld bytes; enough %d\n", (long)target_size, (long)free_space, enough_space); @@ -361,8 +382,8 @@ int main(int argc, char** argv) { made_copy = 1; unlink(source_filename); - size_t free_space = FreeSpaceForFile(source_fs); - printf("(now %ld bytes free for source)\n", (long)free_space); + size_t free_space = FreeSpaceForFile(target_fs); + printf("(now %ld bytes free for target)\n", (long)free_space); } FileContents* source_to_use; @@ -375,14 +396,14 @@ int main(int argc, char** argv) { patch_filename = copy_patch_filename; } - // We write the decoded output to ".patch". - char* outname = (char*)malloc(strlen(source_filename) + 10); - strcpy(outname, source_filename); + // We write the decoded output to ".patch". + char* outname = (char*)malloc(strlen(target_filename) + 10); + strcpy(outname, target_filename); strcat(outname, ".patch"); FILE* output = fopen(outname, "wb"); if (output == NULL) { fprintf(stderr, "failed to patch file %s: %s\n", - source_filename, strerror(errno)); + target_filename, strerror(errno)); return 1; } @@ -441,10 +462,10 @@ int main(int argc, char** argv) { return 1; } - // Finally, rename the .patch file to replace the original source file. - if (rename(outname, source_filename) != 0) { + // Finally, rename the .patch file to replace the target file. + if (rename(outname, target_filename) != 0) { fprintf(stderr, "rename of .patch to \"%s\" failed: %s\n", - source_filename, strerror(errno)); + target_filename, strerror(errno)); return 1; } diff --git a/tools/applypatch/applypatch.sh b/tools/applypatch/applypatch.sh index 181cd5c481..88f3025ff0 100755 --- a/tools/applypatch/applypatch.sh +++ b/tools/applypatch/applypatch.sh @@ -24,16 +24,22 @@ WORK_DIR=/system # partition that WORK_DIR is located on, without the leading slash WORK_FS=system +# set to 0 to use a device instead +USE_EMULATOR=1 + # ------------------------ tmpdir=$(mktemp -d) -emulator -wipe-data -noaudio -no-window -port $EMULATOR_PORT & -pid_emulator=$! +if [ "$USE_EMULATOR" == 1 ]; then + emulator -wipe-data -noaudio -no-window -port $EMULATOR_PORT & + pid_emulator=$! + ADB="adb -s emulator-$EMULATOR_PORT " +else + ADB="adb -d " +fi -ADB="adb -s emulator-$EMULATOR_PORT " - -echo "emulator is $pid_emulator; waiting for startup" +echo "waiting to connect to device" $ADB wait-for-device echo "device is available" $ADB remount @@ -56,7 +62,8 @@ fail() { echo echo FAIL: $testname echo - kill $pid_emulator + [ "$open_pid" == "" ] || kill $open_pid + [ "$pid_emulator" == "" ] || kill $pid_emulator exit 1 } @@ -68,6 +75,23 @@ free_space() { run_command df | awk "/$1/ {print gensub(/K/, \"\", \"g\", \$6)}" } +cleanup() { + # not necessary if we're about to kill the emulator, but nice for + # running on real devices or already-running emulators. + testname "removing test files" + run_command rm $WORK_DIR/bloat.dat + run_command rm $WORK_DIR/old.file + run_command rm $WORK_DIR/patch.bsdiff + run_command rm $WORK_DIR/applypatch + run_command rm $CACHE_TEMP_SOURCE + run_command rm /cache/bloat*.dat + + [ "$pid_emulator" == "" ] || kill $pid_emulator + + rm -rf $tmpdir +} + +cleanup $ADB push $ANDROID_PRODUCT_OUT/system/bin/applypatch $WORK_DIR/applypatch @@ -146,16 +170,71 @@ if (( free_kb * 1024 < NEW_SIZE * 3 / 2 )); then fi testname "apply bsdiff patch" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail testname "reapply bsdiff patch" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail +# --------------- apply patch in new location ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR +$ADB push $DATA_DIR/patch.bsdiff $WORK_DIR + +# Check that the partition has enough space to apply the patch without +# copying. If it doesn't, we'll be testing the low-space condition +# when we intend to test the not-low-space condition. +testname "apply patch to new location (with enough space)" +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS." +if (( free_kb * 1024 < NEW_SIZE * 3 / 2 )); then + echo "Not enough space on /$WORK_FS to patch test file." + echo + echo "This doesn't mean that applypatch is necessarily broken;" + echo "just that /$WORK_FS doesn't have enough free space to" + echo "properly run this test." + exit 1 +fi + +run_command rm $WORK_DIR/new.file +run_command rm $CACHE_TEMP_SOURCE + +testname "apply bsdiff patch to new location" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +testname "reapply bsdiff patch to new location" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +$ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE +# put some junk in the old file +run_command dd if=/dev/urandom of=$WORK_DIR/old.file count=100 bs=1024 || fail + +testname "apply bsdiff patch to new location with corrupted source" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +# put some junk in the cache copy, too +run_command dd if=/dev/urandom of=$CACHE_TEMP_SOURCE count=100 bs=1024 || fail + +run_command rm $WORK_DIR/new.file +testname "apply bsdiff patch to new location with corrupted source and copy (no new file)" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo && fail + +# put some junk in the new file +run_command dd if=/dev/urandom of=$WORK_DIR/new.file count=100 bs=1024 || fail + +testname "apply bsdiff patch to new location with corrupted source and copy (bad new file)" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo && fail + # --------------- apply patch with low space on /system ---------------------- $ADB push $DATA_DIR/old.file $WORK_DIR @@ -169,12 +248,12 @@ free_kb=$(free_space $WORK_FS) echo "${free_kb}kb free on /$WORK_FS now." testname "apply bsdiff patch with low space" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail testname "reapply bsdiff patch with low space" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail @@ -213,7 +292,7 @@ run_command ls /cache/subdir/a.file || fail # wasn't deleted because run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy # should fail; not enough files can be deleted -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff && fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff && fail run_command ls /cache/bloat_large.dat || fail # wasn't deleted because it was open run_command ls /cache/subdir/a.file || fail # wasn't deleted because it's in a subdir run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy @@ -229,7 +308,7 @@ run_command ls /cache/subdir/a.file || fail # still wasn't deleted because i run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy # should succeed -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail run_command ls /cache/subdir/a.file || fail # still wasn't deleted because it's in a subdir @@ -242,7 +321,7 @@ $ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE run_command dd if=/dev/urandom of=$WORK_DIR/old.file count=100 bs=1024 || fail testname "apply bsdiff patch from cache (corrupted source) with low space" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail @@ -251,20 +330,14 @@ $ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE run_command rm $WORK_DIR/old.file testname "apply bsdiff patch from cache (missing source) with low space" -run_command $WORK_DIR/applypatch $WORK_DIR/old.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail $ADB pull $WORK_DIR/old.file $tmpdir/patched diff -q $DATA_DIR/new.file $tmpdir/patched || fail # --------------- cleanup ---------------------- -# not necessary if we're about to kill the emulator, but nice for -# running on real devices or already-running emulators. -run_command rm /cache/bloat*.dat $WORK_DIR/bloat.dat $CACHE_TEMP_SOURCE $WORK_DIR/old.file $WORK_DIR/patch.xdelta3 $WORK_DIR/patch.bsdiff $WORK_DIR/applypatch - -kill $pid_emulator - -rm -rf $tmpdir +cleanup echo echo PASS diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files index c1e377a084..69e9f5b5a6 100755 --- a/tools/releasetools/ota_from_target_files +++ b/tools/releasetools/ota_from_target_files @@ -561,7 +561,7 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip): script.append("show_progress %f 1" % (next_sizes * pb_apply / total_patched_size,)) script.append(("run_program PACKAGE:applypatch " - "/%s %s %d %s:/tmp/patchtmp/%s.p") % + "/%s - %s %d %s:/tmp/patchtmp/%s.p") % (fn, tf.sha1, tf.size, sf.sha1, fn)) target_symlinks = CopySystemFiles(target_zip, None)