Merge "java reader reads directly from map" into main am: 9f254c6b5d am: f4565686e4

Original change: https://android-review.googlesource.com/c/platform/build/+/3251572

Change-Id: I334ca9039f83bfe1e0a3838c5492af6e11f0423f
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java

@@ -16,41 +16,50 @@
 
 package android.aconfig.storage;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Objects;
 
 public class FlagTable {
 
     private Header mHeader;
-    private Map<String, Node> mNodeMap;
+    private ByteBufferReader mReader;
 
     public static FlagTable fromBytes(ByteBuffer bytes) {
         FlagTable flagTable = new FlagTable();
-        ByteBufferReader reader = new ByteBufferReader(bytes);
+        flagTable.mReader = new ByteBufferReader(bytes);
-        Header header = Header.fromBytes(reader);
+        flagTable.mHeader = Header.fromBytes(flagTable.mReader);
-        flagTable.mHeader = header;
+
-        flagTable.mNodeMap = new HashMap(TableUtils.getTableSize(header.mNumFlags));
-        reader.position(header.mNodeOffset);
-        for (int i = 0; i < header.mNumFlags; i++) {
-            Node node = Node.fromBytes(reader);
-            flagTable.mNodeMap.put(makeKey(node.mPackageId, node.mFlagName), node);
-        }
         return flagTable;
     }
 
     public Node get(int packageId, String flagName) {
-        return mNodeMap.get(makeKey(packageId, flagName));
+        int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
+        int bucketIndex = TableUtils.getBucketIndex(makeKey(packageId, flagName), numBuckets);
+
+        mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
+        int nodeIndex = mReader.readInt();
+
+        while (nodeIndex != -1) {
+            mReader.position(nodeIndex);
+            Node node = Node.fromBytes(mReader);
+            if (Objects.equals(flagName, node.mFlagName) && packageId == node.mPackageId) {
+                return node;
+            }
+            nodeIndex = node.mNextOffset;
+        }
+
+        throw new AconfigStorageException("get cannot find flag: " + flagName);
     }
 
     public Header getHeader() {
         return mHeader;
     }
 
-    private static String makeKey(int packageId, String flagName) {
+    private static byte[] makeKey(int packageId, String flagName) {
         StringBuilder ret = new StringBuilder();
-        return ret.append(packageId).append('/').append(flagName).toString();
+        return ret.append(packageId).append('/').append(flagName).toString().getBytes(UTF_8);
     }
 
     public static class Header {

tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java

@@ -16,6 +16,8 @@
 
 package android.aconfig.storage;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
@@ -33,13 +35,22 @@ public class PackageTable {
     }
 
     public Node get(String packageName) {
-        mReader.position(mHeader.mNodeOffset);
+
-        for (int i = 0; i < mHeader.mNumPackages; i++) {
+        int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
+        int bucketIndex = TableUtils.getBucketIndex(packageName.getBytes(UTF_8), numBuckets);
+
+        mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
+        int nodeIndex = mReader.readInt();
+
+        while (nodeIndex != -1) {
+            mReader.position(nodeIndex);
             Node node = Node.fromBytes(mReader);
-            if (Objects.equals(node.mPackageName, packageName)) {
+            if (Objects.equals(packageName, node.mPackageName)) {
                 return node;
             }
+            nodeIndex = node.mNextOffset;
         }
+
         throw new AconfigStorageException("get cannot find package: " + packageName);
     }
 

tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/SipHasher13.java

@@ -44,43 +44,39 @@ public class SipHasher13 {
 
         private void cRounds() {
             v0 += v1;
-            v1 = rotateLeft(v1, 13);
+            v1 = Long.rotateLeft(v1, 13);
             v1 ^= v0;
-            v0 = rotateLeft(v0, 32);
+            v0 = Long.rotateLeft(v0, 32);
             v2 += v3;
-            v3 = rotateLeft(v3, 16);
+            v3 = Long.rotateLeft(v3, 16);
             v3 ^= v2;
             v0 += v3;
-            v3 = rotateLeft(v3, 21);
+            v3 = Long.rotateLeft(v3, 21);
             v3 ^= v0;
             v2 += v1;
-            v1 = rotateLeft(v1, 17);
+            v1 = Long.rotateLeft(v1, 17);
             v1 ^= v2;
-            v2 = rotateLeft(v2, 32);
+            v2 = Long.rotateLeft(v2, 32);
         }
 
         private void dRounds() {
             for (int i = 0; i < 3; i++) {
                 v0 += v1;
-                v1 = rotateLeft(v1, 13);
+                v1 = Long.rotateLeft(v1, 13);
                 v1 ^= v0;
-                v0 = rotateLeft(v0, 32);
+                v0 = Long.rotateLeft(v0, 32);
                 v2 += v3;
-                v3 = rotateLeft(v3, 16);
+                v3 = Long.rotateLeft(v3, 16);
                 v3 ^= v2;
                 v0 += v3;
-                v3 = rotateLeft(v3, 21);
+                v3 = Long.rotateLeft(v3, 21);
                 v3 ^= v0;
                 v2 += v1;
-                v1 = rotateLeft(v1, 17);
+                v1 = Long.rotateLeft(v1, 17);
                 v1 ^= v2;
-                v2 = rotateLeft(v2, 32);
+                v2 = Long.rotateLeft(v2, 32);
             }
         }
-
-        private static long rotateLeft(long value, int shift) {
-            return (value << shift) | value >>> (64 - shift);
-        }
     }
 
     public static long hash(byte[] data) {

tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java

@@ -58,4 +58,9 @@ public class TableUtils {
         }
         throw new AconfigStorageException("Number of items in a hash table exceeds limit");
     }
+
+    public static int getBucketIndex(byte[] val, int numBuckets) {
+        long hashVal = SipHasher13.hash(val);
+        return (int) Long.remainderUnsigned(hashVal, numBuckets);
+    }
 }

tools/aconfig/aconfig_storage_file/tests/Android.bp

@@ -30,9 +30,10 @@ android_test {
     static_libs: [
         "androidx.test.runner",
         "junit",
+        "aconfig_storage_file_java",
     ],
     test_config: "AndroidStorageJaveTest.xml",
-    certificate: "platform",
+    sdk_version: "test_current",
     data: [
         "package.map",
         "flag.map",
@@ -42,4 +43,5 @@ android_test {
     test_suites: [
         "general-tests",
     ],
+    jarjar_rules: "jarjar.txt",
 }

tools/aconfig/aconfig_storage_file/tests/jarjar.txt

@@ -0,0 +1,15 @@
+rule android.aconfig.storage.AconfigStorageException android.aconfig.storage.test.AconfigStorageException
+rule android.aconfig.storage.FlagTable android.aconfig.storage.test.FlagTable
+rule android.aconfig.storage.PackageTable android.aconfig.storage.test.PackageTable
+rule android.aconfig.storage.ByteBufferReader android.aconfig.storage.test.ByteBufferReader
+rule android.aconfig.storage.FlagType android.aconfig.storage.test.FlagType
+rule android.aconfig.storage.SipHasher13 android.aconfig.storage.test.SipHasher13
+rule android.aconfig.storage.FileType android.aconfig.storage.test.FileType
+rule android.aconfig.storage.FlagValueList android.aconfig.storage.test.FlagValueList
+rule android.aconfig.storage.TableUtils android.aconfig.storage.test.TableUtils
+
+
+rule android.aconfig.storage.FlagTable$* android.aconfig.storage.test.FlagTable$@1
+rule android.aconfig.storage.PackageTable$* android.aconfig.storage.test.PackageTable$@1
+rule android.aconfig.storage.FlagValueList$* android.aconfig.storage.test.FlagValueList@1
+rule android.aconfig.storage.SipHasher13$* android.aconfig.storage.test.SipHasher13@1

tools/aconfig/aconfig_storage_read_api/Android.bp

@@ -147,6 +147,7 @@ rust_ffi_shared {
     crate_name: "aconfig_storage_read_api_rust_jni",
     srcs: ["srcs/lib.rs"],
     rustlibs: [
+        "libaconfig_storage_file",
         "libaconfig_storage_read_api",
         "libanyhow",
         "libjni",

tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/AconfigStorageReadAPI.java

@@ -16,18 +16,14 @@
 
 package android.aconfig.storage;
 
+import dalvik.annotation.optimization.FastNative;
+
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-
-import android.aconfig.storage.PackageReadContext;
-import android.aconfig.storage.FlagReadContext;
-
-import dalvik.annotation.optimization.FastNative;
 
 public class AconfigStorageReadAPI {
 
@@ -50,9 +46,8 @@ public class AconfigStorageReadAPI {
     }
 
     // Map a storage file given container and file type
-    public static MappedByteBuffer getMappedFile(
+    public static MappedByteBuffer getMappedFile(String container, StorageFileType type)
-        String container,
+            throws IOException {
-        StorageFileType type) throws IOException{
         switch (type) {
             case PACKAGE_MAP:
                 return mapStorageFile(STORAGEDIR + "/maps/" + container + ".package.map");
@@ -79,7 +74,7 @@ public class AconfigStorageReadAPI {
     // @param mappedFile: memory mapped package map file
     // @param packageName: package name
     // @throws IOException if the passed in file is not a valid package map file
-    static public PackageReadContext getPackageReadContext (
+    public static PackageReadContext getPackageReadContext(
             ByteBuffer mappedFile, String packageName) throws IOException {
         ByteBuffer buffer = getPackageReadContextImpl(mappedFile, packageName);
         buffer.order(ByteOrder.LITTLE_ENDIAN);
@@ -115,8 +110,11 @@ public class AconfigStorageReadAPI {
     // @throws IOException if the passed in file is not a valid flag value file or the
     // flag index went over the file boundary.
     @FastNative
-    public static native boolean getBooleanFlagValue(
+    public static native boolean getBooleanFlagValue(ByteBuffer mappedFile, int flagIndex)
-        ByteBuffer mappedFile, int flagIndex) throws IOException;
+            throws IOException;
+
+    @FastNative
+    public static native long hash(String packageName) throws IOException;
 
     static {
         System.loadLibrary("aconfig_storage_read_api_rust_jni");

tools/aconfig/aconfig_storage_read_api/srcs/lib.rs

@@ -1,5 +1,6 @@
 //! aconfig storage read api java rust interlop
 
+use aconfig_storage_file::SipHasher13;
 use aconfig_storage_read_api::flag_table_query::find_flag_read_context;
 use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value;
 use aconfig_storage_read_api::package_table_query::find_package_read_context;
@@ -7,8 +8,9 @@ use aconfig_storage_read_api::{FlagReadContext, PackageReadContext};
 
 use anyhow::Result;
 use jni::objects::{JByteBuffer, JClass, JString};
-use jni::sys::{jboolean, jint};
+use jni::sys::{jboolean, jint, jlong};
 use jni::JNIEnv;
+use std::hash::Hasher;
 
 /// Call rust find package read context
 fn get_package_read_context_java(
@@ -158,3 +160,30 @@ pub extern "system" fn Java_android_aconfig_storage_AconfigStorageReadAPI_getBoo
         }
     }
 }
+
+/// Get flag value JNI
+#[no_mangle]
+#[allow(unused)]
+pub extern "system" fn Java_android_aconfig_storage_AconfigStorageReadAPI_hash<'local>(
+    mut env: JNIEnv<'local>,
+    class: JClass<'local>,
+    package_name: JString<'local>,
+) -> jlong {
+    match siphasher13_hash(&mut env, package_name) {
+        Ok(value) => value as jlong,
+        Err(errmsg) => {
+            env.throw(("java/io/IOException", errmsg.to_string())).expect("failed to throw");
+            0i64
+        }
+    }
+}
+
+fn siphasher13_hash(env: &mut JNIEnv, package_name: JString) -> Result<u64> {
+    // SAFETY:
+    // The safety here is ensured as the flag name is guaranteed to be a java string
+    let flag_name: String = unsafe { env.get_string_unchecked(&package_name)?.into() };
+    let mut s = SipHasher13::new();
+    s.write(flag_name.as_bytes());
+    s.write_u8(0xff);
+    Ok(s.finish())
+}

tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java

@@ -16,25 +16,26 @@
 
 package android.aconfig.storage.test;
 
-import java.io.IOException;
-import java.nio.MappedByteBuffer;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+
+import android.aconfig.DeviceProtos;
+import android.aconfig.nano.Aconfig.parsed_flag;
+import android.aconfig.storage.AconfigStorageReadAPI;
+import android.aconfig.storage.FlagReadContext;
+import android.aconfig.storage.FlagReadContext.StoredFlagType;
+import android.aconfig.storage.PackageReadContext;
+import android.aconfig.storage.SipHasher13;
+import android.aconfig.storage.StorageInternalReader;
+
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-import android.aconfig.storage.AconfigStorageReadAPI;
+import java.io.IOException;
-import android.aconfig.storage.PackageReadContext;
+import java.nio.MappedByteBuffer;
-import android.aconfig.storage.FlagReadContext;
+import java.util.ArrayList;
-import android.aconfig.storage.FlagReadContext.StoredFlagType;
+import java.util.List;
 
 @RunWith(JUnit4.class)
 public class AconfigStorageReadAPITest {
@@ -45,25 +46,28 @@ public class AconfigStorageReadAPITest{
     public void testPackageContextQuery() {
         MappedByteBuffer packageMap = null;
         try {
-            packageMap = AconfigStorageReadAPI.mapStorageFile(
+            packageMap =
-                mStorageDir + "/maps/mockup.package.map");
+                    AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
         assertTrue(packageMap != null);
 
         try {
-            PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
+            PackageReadContext context =
+                    AconfigStorageReadAPI.getPackageReadContext(
                             packageMap, "com.android.aconfig.storage.test_1");
             assertEquals(context.mPackageId, 0);
             assertEquals(context.mBooleanStartIndex, 0);
 
-            context = AconfigStorageReadAPI.getPackageReadContext(
+            context =
+                    AconfigStorageReadAPI.getPackageReadContext(
                             packageMap, "com.android.aconfig.storage.test_2");
             assertEquals(context.mPackageId, 1);
             assertEquals(context.mBooleanStartIndex, 3);
 
-            context = AconfigStorageReadAPI.getPackageReadContext(
+            context =
+                    AconfigStorageReadAPI.getPackageReadContext(
                             packageMap, "com.android.aconfig.storage.test_4");
             assertEquals(context.mPackageId, 2);
             assertEquals(context.mBooleanStartIndex, 6);
@@ -76,16 +80,16 @@ public class AconfigStorageReadAPITest{
     public void testNonExistPackageContextQuery() {
         MappedByteBuffer packageMap = null;
         try {
-            packageMap = AconfigStorageReadAPI.mapStorageFile(
+            packageMap =
-                mStorageDir + "/maps/mockup.package.map");
+                    AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
         assertTrue(packageMap != null);
 
         try {
-            PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
+            PackageReadContext context =
-                packageMap, "unknown");
+                    AconfigStorageReadAPI.getPackageReadContext(packageMap, "unknown");
             assertEquals(context.mPackageId, -1);
             assertEquals(context.mBooleanStartIndex, -1);
         } catch (IOException ex) {
@@ -97,8 +101,7 @@ public class AconfigStorageReadAPITest{
     public void testFlagContextQuery() {
         MappedByteBuffer flagMap = null;
         try {
-            flagMap = AconfigStorageReadAPI.mapStorageFile(
+            flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
-                mStorageDir + "/maps/mockup.flag.map");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
@@ -110,10 +113,8 @@ public class AconfigStorageReadAPITest{
             public StoredFlagType mFlagType;
             public int mFlagIndex;
 
-            public Baseline(int packageId,
+            public Baseline(
-                    String flagName,
+                    int packageId, String flagName, StoredFlagType flagType, int flagIndex) {
-                    StoredFlagType flagType,
-                    int flagIndex) {
                 mPackageId = packageId;
                 mFlagName = flagName;
                 mFlagType = flagType;
@@ -133,7 +134,8 @@ public class AconfigStorageReadAPITest{
 
         try {
             for (Baseline baseline : baselines) {
-                FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
+                FlagReadContext context =
+                        AconfigStorageReadAPI.getFlagReadContext(
                                 flagMap, baseline.mPackageId, baseline.mFlagName);
                 assertEquals(context.mFlagType, baseline.mFlagType);
                 assertEquals(context.mFlagIndex, baseline.mFlagIndex);
@@ -147,21 +149,19 @@ public class AconfigStorageReadAPITest{
     public void testNonExistFlagContextQuery() {
         MappedByteBuffer flagMap = null;
         try {
-            flagMap = AconfigStorageReadAPI.mapStorageFile(
+            flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
-                mStorageDir + "/maps/mockup.flag.map");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
         assertTrue(flagMap != null);
 
         try {
-            FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
+            FlagReadContext context =
-                flagMap, 0,  "unknown");
+                    AconfigStorageReadAPI.getFlagReadContext(flagMap, 0, "unknown");
             assertEquals(context.mFlagType, null);
             assertEquals(context.mFlagIndex, -1);
 
-            context = AconfigStorageReadAPI.getFlagReadContext(
+            context = AconfigStorageReadAPI.getFlagReadContext(flagMap, 3, "enabled_ro");
-                flagMap, 3,  "enabled_ro");
             assertEquals(context.mFlagType, null);
             assertEquals(context.mFlagIndex, -1);
         } catch (IOException ex) {
@@ -173,8 +173,7 @@ public class AconfigStorageReadAPITest{
     public void testBooleanFlagValueQuery() {
         MappedByteBuffer flagVal = null;
         try {
-            flagVal = AconfigStorageReadAPI.mapStorageFile(
+            flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
-                mStorageDir + "/boot/mockup.val");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
@@ -195,8 +194,7 @@ public class AconfigStorageReadAPITest{
     public void testInvalidBooleanFlagValueQuery() {
         MappedByteBuffer flagVal = null;
         try {
-            flagVal = AconfigStorageReadAPI.mapStorageFile(
+            flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
-                mStorageDir + "/boot/mockup.val");
         } catch (IOException ex) {
             assertTrue(ex.toString(), false);
         }
@@ -210,4 +208,63 @@ public class AconfigStorageReadAPITest{
             assertTrue(ex.toString(), ex.toString().contains(expectedErrmsg));
         }
     }
+
+    @Test
+    public void testRustJavaEqualHash() throws IOException {
+        List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
+        for (parsed_flag flag : flags) {
+            String packageName = flag.package_;
+            String flagName = flag.name;
+            long rHash = AconfigStorageReadAPI.hash(packageName);
+            long jHash = SipHasher13.hash(packageName.getBytes());
+            assertEquals(rHash, jHash);
+
+            String fullFlagName = packageName + "/" + flagName;
+            rHash = AconfigStorageReadAPI.hash(fullFlagName);
+            jHash = SipHasher13.hash(fullFlagName.getBytes());
+            assertEquals(rHash, jHash);
+        }
+    }
+
+    @Test
+    public void testRustJavaEqualFlag() throws IOException {
+        List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
+
+        String mapPath = "/metadata/aconfig/maps/";
+        String flagsPath = "/metadata/aconfig/boot/";
+
+        for (parsed_flag flag : flags) {
+
+            String container = flag.container;
+            String packageName = flag.package_;
+            String flagName = flag.name;
+            String fullFlagName = packageName + "/" + flagName;
+
+            MappedByteBuffer packageMap =
+                    AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".package.map");
+            MappedByteBuffer flagMap =
+                    AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".flag.map");
+            MappedByteBuffer flagValList =
+                    AconfigStorageReadAPI.mapStorageFile(flagsPath + container + ".val");
+
+            PackageReadContext packageContext =
+                    AconfigStorageReadAPI.getPackageReadContext(packageMap, packageName);
+
+            FlagReadContext flagContext =
+                    AconfigStorageReadAPI.getFlagReadContext(
+                            flagMap, packageContext.mPackageId, flagName);
+
+            boolean rVal =
+                    AconfigStorageReadAPI.getBooleanFlagValue(
+                            flagValList,
+                            packageContext.mBooleanStartIndex + flagContext.mFlagIndex);
+
+            StorageInternalReader reader = new StorageInternalReader(container, packageName);
+            boolean jVal = reader.getBooleanFlagValue(flagContext.mFlagIndex);
+
+            long rHash = AconfigStorageReadAPI.hash(packageName);
+            long jHash = SipHasher13.hash(packageName.getBytes());
+            assertEquals(rVal, jVal);
+        }
+    }
 }

tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp

@@ -2,6 +2,8 @@ android_test {
     name: "aconfig_storage_read_api.test.java",
     srcs: ["./**/*.java"],
     static_libs: [
+        "aconfig_device_paths_java",
+        "aconfig_storage_file_java",
         "aconfig_storage_reader_java",
         "androidx.test.rules",
         "libaconfig_storage_read_api_java",