Merge change 2951 into donut

* changes:
  support incremental updates of boot image

tools/applypatch/applypatch.c

@@ -27,9 +27,12 @@
 #include "applypatch.h"
 #include "mtdutils/mtdutils.h"
 
+int SaveFileContents(const char* filename, FileContents file);
 int LoadMTDContents(const char* filename, FileContents* file);
 int ParseSha1(const char* str, uint8_t* digest);
 
+static int mtd_partitions_scanned = 0;
+
 // Read a file into memory; store it and its associated metadata in
 // *file.  Return 0 on success.
 int LoadFileContents(const char* filename, FileContents* file) {
@@ -139,15 +142,14 @@ int LoadMTDContents(const char* filename, FileContents* file) {
     index[i] = i;
   }
 
-  // sort the index[] array so it indexs the pairs in order of
+  // sort the index[] array so it indexes the pairs in order of
   // increasing size.
   size_array = size;
   qsort(index, pairs, sizeof(int), compare_size_indices);
 
-  static int partitions_scanned = 0;
+  if (!mtd_partitions_scanned) {
-  if (!partitions_scanned) {
     mtd_scan_partitions();
-    partitions_scanned = 1;
+    mtd_partitions_scanned = 1;
   }
 
   const MtdPartition* mtd = mtd_find_partition_by_name(partition);
@@ -234,6 +236,11 @@ int LoadMTDContents(const char* filename, FileContents* file) {
     file->sha1[i] = sha_final[i];
   }
 
+  // Fake some stat() info.
+  file->st.st_mode = 0644;
+  file->st.st_uid = 0;
+  file->st.st_gid = 0;
+
   free(copy);
   free(index);
   free(size);
@@ -275,6 +282,76 @@ int SaveFileContents(const char* filename, FileContents file) {
   return 0;
 }
 
+// Copy the contents of source_file to target_mtd partition, a string
+// of the form "MTD:<partition>[:...]".  Return 0 on success.
+int CopyToMTDPartition(const char* source_file, const char* target_mtd) {
+  char* partition = strchr(target_mtd, ':');
+  if (partition == NULL) {
+    fprintf(stderr, "bad MTD target name \"%s\"\n", target_mtd);
+    return -1;
+  }
+  ++partition;
+  // Trim off anything after a colon, eg "MTD:boot:blah:blah:blah...".
+  // We want just the partition name "boot".
+  partition = strdup(partition);
+  char* end = strchr(partition, ':');
+  if (end != NULL)
+    *end = '\0';
+
+  FILE* f = fopen(source_file, "rb");
+  if (f == NULL) {
+    fprintf(stderr, "failed to open %s for reading: %s\n",
+            source_file, strerror(errno));
+    return -1;
+  }
+
+  if (!mtd_partitions_scanned) {
+    mtd_scan_partitions();
+    mtd_partitions_scanned = 1;
+  }
+
+  const MtdPartition* mtd = mtd_find_partition_by_name(partition);
+  if (mtd == NULL) {
+    fprintf(stderr, "mtd partition \"%s\" not found for writing\n", partition);
+    return -1;
+  }
+
+  MtdWriteContext* ctx = mtd_write_partition(mtd);
+  if (ctx == NULL) {
+    fprintf(stderr, "failed to init mtd partition \"%s\" for writing\n",
+            partition);
+    return -1;
+  }
+
+  const int buffer_size = 4096;
+  char buffer[buffer_size];
+  size_t read;
+  while ((read = fread(buffer, 1, buffer_size, f)) > 0) {
+    size_t written = mtd_write_data(ctx, buffer, read);
+    if (written != read) {
+      fprintf(stderr, "only wrote %d of %d bytes to MTD %s\n",
+              written, read, partition);
+      mtd_write_close(ctx);
+      return -1;
+    }
+  }
+
+  fclose(f);
+  if (mtd_erase_blocks(ctx, -1) < 0) {
+    fprintf(stderr, "error finishing mtd write of %s\n", partition);
+    mtd_write_close(ctx);
+    return -1;
+  }
+
+  if (mtd_write_close(ctx)) {
+    fprintf(stderr, "error closing mtd write of %s\n", partition);
+    return -1;
+  }
+
+  free(partition);
+  return 0;
+}
+
 
 // Take a string 'str' of 40 hex digits and parse it into the 20
 // byte array 'digest'.  'str' may contain only the digest or be of
@@ -443,9 +520,10 @@ int main(int argc, char** argv) {
             "   or  %s -s <bytes>\n"
             "   or  %s -l\n"
             "\n"
-            "<src-file> or <file> may be of the form\n"
+            "Filenames may be of the form\n"
-            "  MTD:<partition>:<len_1>:<sha1_1>:<len_2>:<sha1_2>:...\n"
+            "  MTD:<partition>:<len_1>:<sha1_1>:<len_2>:<sha1_2>"
-            "to specify reading from an MTD partition.\n\n",
+              ":...:<backup-file>\n"
+            "to specify reading from or writing to an MTD partition.\n\n",
             argv[0], argv[0], argv[0], argv[0]);
     return 1;
   }
@@ -480,16 +558,7 @@ int main(int argc, char** argv) {
     target_filename = source_filename;
   }
 
-  // assume that target_filename (eg "/system/app/Foo.apk") is located
+ if (ParseSha1(argv[3], target_sha1) != 0) {
-  // on the same filesystem as its top-level directory ("/system").
-  // We need something that exists for calling statfs().
-  char* target_fs = strdup(target_filename);
-  char* slash = strchr(target_fs+1, '/');
-  if (slash != NULL) {
-    *slash = '\0';
-  }
-
-  if (ParseSha1(argv[3], target_sha1) != 0) {
     fprintf(stderr, "failed to parse tgt-sha1 \"%s\"\n", argv[3]);
     return 1;
   }
@@ -557,39 +626,70 @@ int main(int argc, char** argv) {
     }
   }
 
-  // Is there enough room in the target filesystem to hold the patched file?
+  // Is there enough room in the target filesystem to hold the patched
-  size_t free_space = FreeSpaceForFile(target_fs);
+  // file?
-  int enough_space = free_space > (target_size * 3 / 2);  // 50% margin of error
-  printf("target %ld bytes; free space %ld bytes; enough %d\n",
-         (long)target_size, (long)free_space, enough_space);
 
-  if (!enough_space && source_patch_filename != NULL) {
+  if (strncmp(target_filename, "MTD:", 4) == 0) {
-    // Using the original source, but not enough free space.  First
+    // If the target is an MTD partition, we're actually going to
-    // copy the source file to cache, then delete it from the original
+    // write the output to /tmp and then copy it to the partition.
-    // location.
+    // statfs() always returns 0 blocks free for /tmp, so instead
-
+    // we'll just assume that /tmp has enough space to hold the file.
-    if (strncmp(source_filename, "MTD:", 4) == 0) {
-      // It's impossible to free space on the target filesystem by
-      // deleting the source if the source is an MTD partition.  If
-      // we're ever in a state where we need to do this, fail.
-      fprintf(stderr, "not enough free space for target but source is MTD\n");
-      return 1;
-    }
 
+    // We still write the original source to cache, in case the MTD
+    // write is interrupted.
     if (MakeFreeSpaceOnCache(source_file.size) < 0) {
       fprintf(stderr, "not enough free space on /cache\n");
       return 1;
     }
-
     if (SaveFileContents(CACHE_TEMP_SOURCE, source_file) < 0) {
       fprintf(stderr, "failed to back up source file\n");
       return 1;
     }
     made_copy = 1;
-    unlink(source_filename);
+  } else {
+    // assume that target_filename (eg "/system/app/Foo.apk") is located
+    // on the same filesystem as its top-level directory ("/system").
+    // We need something that exists for calling statfs().
+    char* target_fs = strdup(target_filename);
+    char* slash = strchr(target_fs+1, '/');
+    if (slash != NULL) {
+      *slash = '\0';
+    }
 
     size_t free_space = FreeSpaceForFile(target_fs);
-    printf("(now %ld bytes free for target)\n", (long)free_space);
+    int enough_space =
+        free_space > (target_size * 3 / 2);  // 50% margin of error
+    printf("target %ld bytes; free space %ld bytes; enough %d\n",
+           (long)target_size, (long)free_space, enough_space);
+
+    if (!enough_space && source_patch_filename != NULL) {
+      // Using the original source, but not enough free space.  First
+      // copy the source file to cache, then delete it from the original
+      // location.
+
+      if (strncmp(source_filename, "MTD:", 4) == 0) {
+        // It's impossible to free space on the target filesystem by
+        // deleting the source if the source is an MTD partition.  If
+        // we're ever in a state where we need to do this, fail.
+        fprintf(stderr, "not enough free space for target but source is MTD\n");
+        return 1;
+      }
+
+      if (MakeFreeSpaceOnCache(source_file.size) < 0) {
+        fprintf(stderr, "not enough free space on /cache\n");
+        return 1;
+      }
+
+      if (SaveFileContents(CACHE_TEMP_SOURCE, source_file) < 0) {
+        fprintf(stderr, "failed to back up source file\n");
+        return 1;
+      }
+      made_copy = 1;
+      unlink(source_filename);
+
+      size_t free_space = FreeSpaceForFile(target_fs);
+      printf("(now %ld bytes free for target)\n", (long)free_space);
+    }
   }
 
   FileContents* source_to_use;
@@ -602,14 +702,19 @@ int main(int argc, char** argv) {
     patch_filename = copy_patch_filename;
   }
 
-  // We write the decoded output to "<tgt-file>.patch".
+  char* outname = NULL;
-  char* outname = (char*)malloc(strlen(target_filename) + 10);
+  if (strncmp(target_filename, "MTD:", 4) == 0) {
-  strcpy(outname, target_filename);
+    outname = MTD_TARGET_TEMP_FILE;
-  strcat(outname, ".patch");
+  } else {
+    // We write the decoded output to "<tgt-file>.patch".
+    outname = (char*)malloc(strlen(target_filename) + 10);
+    strcpy(outname, target_filename);
+    strcat(outname, ".patch");
+  }
   FILE* output = fopen(outname, "wb");
   if (output == NULL) {
-    fprintf(stderr, "failed to patch file %s: %s\n",
+    fprintf(stderr, "failed to open output file %s: %s\n",
-            target_filename, strerror(errno));
+            outname, strerror(errno));
     return 1;
   }
 
@@ -665,22 +770,32 @@ int main(int argc, char** argv) {
     return 1;
   }
 
-  // Give the .patch file the same owner, group, and mode of the
+  if (strcmp(outname, MTD_TARGET_TEMP_FILE) == 0) {
-  // original source file.
+    // Copy the temp file to the MTD partition.
-  if (chmod(outname, source_to_use->st.st_mode) != 0) {
+    if (CopyToMTDPartition(outname, target_filename) != 0) {
-    fprintf(stderr, "chmod of \"%s\" failed: %s\n", outname, strerror(errno));
+      fprintf(stderr, "copy of %s to %s failed\n", outname, target_filename);
-    return 1;
+      return 1;
-  }
+    }
-  if (chown(outname, source_to_use->st.st_uid, source_to_use->st.st_gid) != 0) {
+    unlink(outname);
-    fprintf(stderr, "chown of \"%s\" failed: %s\n", outname, strerror(errno));
+  } else {
-    return 1;
+    // Give the .patch file the same owner, group, and mode of the
-  }
+    // original source file.
+    if (chmod(outname, source_to_use->st.st_mode) != 0) {
+      fprintf(stderr, "chmod of \"%s\" failed: %s\n", outname, strerror(errno));
+      return 1;
+    }
+    if (chown(outname, source_to_use->st.st_uid,
+              source_to_use->st.st_gid) != 0) {
+      fprintf(stderr, "chown of \"%s\" failed: %s\n", outname, strerror(errno));
+      return 1;
+    }
 
-  // Finally, rename the .patch file to replace the target file.
+    // Finally, rename the .patch file to replace the target file.
-  if (rename(outname, target_filename) != 0) {
+    if (rename(outname, target_filename) != 0) {
-    fprintf(stderr, "rename of .patch to \"%s\" failed: %s\n",
+      fprintf(stderr, "rename of .patch to \"%s\" failed: %s\n",
-            target_filename, strerror(errno));
+              target_filename, strerror(errno));
-    return 1;
+      return 1;
+    }
   }
 
   // If this run of applypatch created the copy, and we're here, we

tools/applypatch/applypatch.h

@@ -38,6 +38,11 @@ typedef struct _FileContents {
 // and use it as the source instead.
 #define CACHE_TEMP_SOURCE "/cache/saved.file"
 
+// When writing to an MTD partition, we first put the output in this
+// temp file, then copy it to the partition once the patching is
+// finished (and the target sha1 verified).
+#define MTD_TARGET_TEMP_FILE "/tmp/mtd-temp"
+
 // applypatch.c
 size_t FreeSpaceForFile(const char* filename);
 

tools/releasetools/ota_from_target_files

@@ -427,11 +427,12 @@ def Difference(tf, sf, diff_program):
     cmd.append(ptemp.name)
     p = common.Run(cmd)
     _, err = p.communicate()
-    if err:
+    if err or p.returncode != 0:
-      raise ExternalError("failure running %s:\n%s\n" % (diff_program, err))
+      print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
+      return None
     diff = ptemp.read()
-    ptemp.close()
   finally:
+    ptemp.close()
     stemp.close()
     ttemp.close()
 
@@ -478,8 +479,9 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
       if tf.name.endswith(".gz"):
         diff_method = "imgdiff"
       d = Difference(tf, sf, diff_method)
-      print fn, tf.size, len(d), (float(len(d)) / tf.size)
+      if d is not None:
-      if len(d) > tf.size * OPTIONS.patch_threshold:
+        print fn, tf.size, len(d), (float(len(d)) / tf.size)
+      if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
         # patch is almost as big as the file; don't bother patching
         tf.AddToZip(output_zip)
         verbatim_targets.append((fn, tf.size))
@@ -503,11 +505,13 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
                  '"ro.build.fingerprint=%s") == "true"') %
                 (source_fp, target_fp))
 
-  source_boot = common.BuildBootableImage(
+  source_boot = File("/tmp/boot.img",
-      os.path.join(OPTIONS.source_tmp, "BOOT"))
+                     common.BuildBootableImage(
-  target_boot = common.BuildBootableImage(
+      os.path.join(OPTIONS.source_tmp, "BOOT")))
-      os.path.join(OPTIONS.target_tmp, "BOOT"))
+  target_boot = File("/tmp/boot.img",
-  updating_boot = (source_boot != target_boot)
+                     common.BuildBootableImage(
+      os.path.join(OPTIONS.target_tmp, "BOOT")))
+  updating_boot = (source_boot.data != target_boot.data)
 
   source_recovery = File("system/recovery.img",
                          common.BuildBootableImage(
@@ -543,12 +547,6 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
     script.append("run_program PACKAGE:applypatch -c /%s %s %s" %
                   (fn, tf.sha1, sf.sha1))
 
-  if patch_list:
-    script.append("run_program PACKAGE:applypatch -s %d" %
-                  (largest_source_size,))
-    script.append("copy_dir PACKAGE:patch CACHE:../tmp/patchtmp")
-    IncludeBinary("applypatch", target_zip, output_zip)
-
   if updating_recovery:
     d = Difference(target_recovery, source_recovery, "imgdiff")
     print "recovery  target: %d  source: %d  diff: %d" % (
@@ -561,6 +559,23 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
                   (source_recovery.size, source_recovery.sha1,
                    target_recovery.size, target_recovery.sha1))
 
+  if updating_boot:
+    d = Difference(target_boot, source_boot, "imgdiff")
+    print "boot      target: %d  source: %d  diff: %d" % (
+        target_boot.size, source_boot.size, len(d))
+
+    output_zip.writestr("patch/boot.img.p", d)
+
+    script.append(("run_program PACKAGE:applypatch -c "
+                   "MTD:boot:%d:%s:%d:%s") %
+                  (source_boot.size, source_boot.sha1,
+                   target_boot.size, target_boot.sha1))
+
+  if patch_list or updating_recovery or updating_boot:
+    script.append("run_program PACKAGE:applypatch -s %d" %
+                  (largest_source_size,))
+    script.append("copy_dir PACKAGE:patch CACHE:../tmp/patchtmp")
+    IncludeBinary("applypatch", target_zip, output_zip)
 
   script.append("\n# ---- start making changes here\n")
 
@@ -570,8 +585,17 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
   DeleteFiles(script, [SubstituteRoot(i[0]) for i in verbatim_targets])
 
   if updating_boot:
-    script.append("format BOOT:")
+    # Produce the boot image by applying a patch to the current
-    output_zip.writestr("boot.img", target_boot)
+    # contents of the boot partition, and write it back to the
+    # partition.
+    script.append(("run_program PACKAGE:applypatch "
+                   "MTD:boot:%d:%s:%d:%s - "
+                   "%s %d %s:/tmp/patchtmp/boot.img.p")
+                  % (source_boot.size, source_boot.sha1,
+                     target_boot.size, target_boot.sha1,
+                     target_boot.sha1,
+                     target_boot.size,
+                     source_boot.sha1))
     print "boot image changed; including."
   else:
     print "boot image unchanged; skipping."
@@ -654,10 +678,6 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
   # permissions.
   script.extend(temp_script)
 
-  if updating_boot:
-    script.append("show_progress 0.1 5")
-    script.append("write_raw_image PACKAGE:boot.img BOOT:")
-
   if OPTIONS.extra_script is not None:
     script.append(OPTIONS.extra_script)