releasetools: Make BBOTA incremental generation repeatable.

set() doesn't keep elements according to the order of insertion. So
Transfers managed with set() in intermediate steps may not appear in the
same order across runs. This leads to slightly different output packages
when generating the same incremental OTA.

This CL fixes the issue by replacing set() with OrderedDict() in
blockimgdiff.GenerateDigraph() and blockimgdiff.FindVertexSequence().

It also adds a testcase that ensures blockimgdiff.GenerateDigraph()
preserves the insertion order for Transfer.goes_after set.

Bug: 32220816
Test: ota_from_target_files.py gives identical package when running
multiple times.

Change-Id: I56d551e5ca926993ab46896e33c80e0ce42e506a

tools/releasetools/blockimgdiff.py

@@ -999,8 +999,11 @@ class BlockImageDiff(object):
       heap.append(xf.heap_item)
     heapq.heapify(heap)
 
-    sinks = set(u for u in G if not u.outgoing)
+    # Use OrderedDict() instead of set() to preserve the insertion order. Need
-    sources = set(u for u in G if not u.incoming)
+    # to use 'sinks[key] = None' to add key into the set. sinks will look like
+    # { key1: None, key2: None, ... }.
+    sinks = OrderedDict.fromkeys(u for u in G if not u.outgoing)
+    sources = OrderedDict.fromkeys(u for u in G if not u.incoming)
 
     def adjust_score(iu, delta):
       iu.score += delta
@@ -1011,26 +1014,28 @@ class BlockImageDiff(object):
     while G:
       # Put all sinks at the end of the sequence.
       while sinks:
-        new_sinks = set()
+        new_sinks = OrderedDict()
         for u in sinks:
           if u not in G: continue
           s2.appendleft(u)
           del G[u]
           for iu in u.incoming:
             adjust_score(iu, -iu.outgoing.pop(u))
-            if not iu.outgoing: new_sinks.add(iu)
+            if not iu.outgoing:
+              new_sinks[iu] = None
         sinks = new_sinks
 
       # Put all the sources at the beginning of the sequence.
       while sources:
-        new_sources = set()
+        new_sources = OrderedDict()
         for u in sources:
           if u not in G: continue
           s1.append(u)
           del G[u]
           for iu in u.outgoing:
             adjust_score(iu, +iu.incoming.pop(u))
-            if not iu.incoming: new_sources.add(iu)
+            if not iu.incoming:
+              new_sources[iu] = None
         sources = new_sources
 
       if not G: break
@@ -1049,11 +1054,13 @@ class BlockImageDiff(object):
       del G[u]
       for iu in u.outgoing:
         adjust_score(iu, +iu.incoming.pop(u))
-        if not iu.incoming: sources.add(iu)
+        if not iu.incoming:
+          sources[iu] = None
 
       for iu in u.incoming:
         adjust_score(iu, -iu.outgoing.pop(u))
-        if not iu.outgoing: sinks.add(iu)
+        if not iu.outgoing:
+          sinks[iu] = None
 
     # Now record the sequence in the 'order' field of each transfer,
     # and by rearranging self.transfers to be in the chosen sequence.
@@ -1072,8 +1079,7 @@ class BlockImageDiff(object):
 
     # Each item of source_ranges will be:
     #   - None, if that block is not used as a source,
-    #   - a transfer, if one transfer uses it as a source, or
+    #   - an ordered set of transfers.
-    #   - a set of transfers.
     source_ranges = []
     for b in self.transfers:
       for s, e in b.src_ranges:
@@ -1081,23 +1087,19 @@ class BlockImageDiff(object):
           source_ranges.extend([None] * (e-len(source_ranges)))
         for i in range(s, e):
           if source_ranges[i] is None:
-            source_ranges[i] = b
+            source_ranges[i] = OrderedDict.fromkeys([b])
           else:
-            if not isinstance(source_ranges[i], set):
+            source_ranges[i][b] = None
-              source_ranges[i] = set([source_ranges[i]])
-            source_ranges[i].add(b)
 
     for a in self.transfers:
-      intersections = set()
+      intersections = OrderedDict()
       for s, e in a.tgt_ranges:
         for i in range(s, e):
           if i >= len(source_ranges): break
-          b = source_ranges[i]
+          # Add all the Transfers in source_ranges[i] to the (ordered) set.
-          if b is not None:
+          if source_ranges[i] is not None:
-            if isinstance(b, set):
+            for j in source_ranges[i]:
-              intersections.update(b)
+              intersections[j] = None
-            else:
-              intersections.add(b)
 
       for b in intersections:
         if a is b: continue

tools/releasetools/test_blockimgdiff.py

@@ -0,0 +1,77 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import print_function
+
+import common
+import unittest
+
+from collections import OrderedDict
+from blockimgdiff import BlockImageDiff, EmptyImage, DataImage, Transfer
+from rangelib import RangeSet
+
+class BlockImageDiffTest(unittest.TestCase):
+
+  def test_GenerateDigraphOrder(self):
+    """Make sure GenerateDigraph preserves the order.
+
+    t0: <0-5> => <...>
+    t1: <0-7> => <...>
+    t2: <0-4> => <...>
+    t3: <...> => <0-10>
+
+    t0, t1 and t2 must go before t3, i.e. t3.goes_after =
+    { t0:..., t1:..., t2:... }. But the order of t0-t2 must be preserved.
+    """
+
+    src = EmptyImage()
+    tgt = EmptyImage()
+    block_image_diff = BlockImageDiff(tgt, src)
+
+    transfers = block_image_diff.transfers
+    t0 = Transfer(
+        "t1", "t1", RangeSet("10-15"), RangeSet("0-5"), "move", transfers)
+    t1 = Transfer(
+        "t2", "t2", RangeSet("20-25"), RangeSet("0-7"), "move", transfers)
+    t2 = Transfer(
+        "t3", "t3", RangeSet("30-35"), RangeSet("0-4"), "move", transfers)
+    t3 = Transfer(
+        "t4", "t4", RangeSet("0-10"), RangeSet("40-50"), "move", transfers)
+
+    block_image_diff.GenerateDigraph()
+    t3_goes_after_copy = t3.goes_after.copy()
+
+    # Elements in the set must be in the transfer evaluation order.
+    elements = list(t3_goes_after_copy)
+    self.assertEqual(t0, elements[0])
+    self.assertEqual(t1, elements[1])
+    self.assertEqual(t2, elements[2])
+
+    # Now switch the order of t0, t1 and t2.
+    transfers[0], transfers[1], transfers[2] = (
+        transfers[2], transfers[0], transfers[1])
+    t3.goes_after.clear()
+    t3.goes_before.clear()
+    block_image_diff.GenerateDigraph()
+
+    # The goes_after must be different from last run.
+    self.assertNotEqual(t3_goes_after_copy, t3.goes_after)
+
+    # Assert that each element must agree with the transfer order.
+    elements = list(t3.goes_after)
+    self.assertEqual(t2, elements[0])
+    self.assertEqual(t0, elements[1])
+    self.assertEqual(t1, elements[2])