Merge "java reader reads directly from map" into main am: 9f254c6b5d am: f4565686e4

Original change: https://android-review.googlesource.com/c/platform/build/+/3251572

Change-Id: I334ca9039f83bfe1e0a3838c5492af6e11f0423f
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
Treehugger Robot
2024-08-30 21:23:21 +00:00
committed by Automerger Merge Worker
11 changed files with 231 additions and 106 deletions

View File

@@ -16,41 +16,50 @@
package android.aconfig.storage;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
public class FlagTable {
private Header mHeader;
private Map<String, Node> mNodeMap;
private ByteBufferReader mReader;
public static FlagTable fromBytes(ByteBuffer bytes) {
FlagTable flagTable = new FlagTable();
ByteBufferReader reader = new ByteBufferReader(bytes);
Header header = Header.fromBytes(reader);
flagTable.mHeader = header;
flagTable.mNodeMap = new HashMap(TableUtils.getTableSize(header.mNumFlags));
reader.position(header.mNodeOffset);
for (int i = 0; i < header.mNumFlags; i++) {
Node node = Node.fromBytes(reader);
flagTable.mNodeMap.put(makeKey(node.mPackageId, node.mFlagName), node);
}
flagTable.mReader = new ByteBufferReader(bytes);
flagTable.mHeader = Header.fromBytes(flagTable.mReader);
return flagTable;
}
public Node get(int packageId, String flagName) {
return mNodeMap.get(makeKey(packageId, flagName));
int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
int bucketIndex = TableUtils.getBucketIndex(makeKey(packageId, flagName), numBuckets);
mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
int nodeIndex = mReader.readInt();
while (nodeIndex != -1) {
mReader.position(nodeIndex);
Node node = Node.fromBytes(mReader);
if (Objects.equals(flagName, node.mFlagName) && packageId == node.mPackageId) {
return node;
}
nodeIndex = node.mNextOffset;
}
throw new AconfigStorageException("get cannot find flag: " + flagName);
}
public Header getHeader() {
return mHeader;
}
private static String makeKey(int packageId, String flagName) {
private static byte[] makeKey(int packageId, String flagName) {
StringBuilder ret = new StringBuilder();
return ret.append(packageId).append('/').append(flagName).toString();
return ret.append(packageId).append('/').append(flagName).toString().getBytes(UTF_8);
}
public static class Header {

View File

@@ -16,6 +16,8 @@
package android.aconfig.storage;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.nio.ByteBuffer;
import java.util.Objects;
@@ -33,13 +35,22 @@ public class PackageTable {
}
public Node get(String packageName) {
mReader.position(mHeader.mNodeOffset);
for (int i = 0; i < mHeader.mNumPackages; i++) {
int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
int bucketIndex = TableUtils.getBucketIndex(packageName.getBytes(UTF_8), numBuckets);
mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
int nodeIndex = mReader.readInt();
while (nodeIndex != -1) {
mReader.position(nodeIndex);
Node node = Node.fromBytes(mReader);
if (Objects.equals(node.mPackageName, packageName)) {
if (Objects.equals(packageName, node.mPackageName)) {
return node;
}
nodeIndex = node.mNextOffset;
}
throw new AconfigStorageException("get cannot find package: " + packageName);
}

View File

@@ -44,43 +44,39 @@ public class SipHasher13 {
private void cRounds() {
v0 += v1;
v1 = rotateLeft(v1, 13);
v1 = Long.rotateLeft(v1, 13);
v1 ^= v0;
v0 = rotateLeft(v0, 32);
v0 = Long.rotateLeft(v0, 32);
v2 += v3;
v3 = rotateLeft(v3, 16);
v3 = Long.rotateLeft(v3, 16);
v3 ^= v2;
v0 += v3;
v3 = rotateLeft(v3, 21);
v3 = Long.rotateLeft(v3, 21);
v3 ^= v0;
v2 += v1;
v1 = rotateLeft(v1, 17);
v1 = Long.rotateLeft(v1, 17);
v1 ^= v2;
v2 = rotateLeft(v2, 32);
v2 = Long.rotateLeft(v2, 32);
}
private void dRounds() {
for (int i = 0; i < 3; i++) {
v0 += v1;
v1 = rotateLeft(v1, 13);
v1 = Long.rotateLeft(v1, 13);
v1 ^= v0;
v0 = rotateLeft(v0, 32);
v0 = Long.rotateLeft(v0, 32);
v2 += v3;
v3 = rotateLeft(v3, 16);
v3 = Long.rotateLeft(v3, 16);
v3 ^= v2;
v0 += v3;
v3 = rotateLeft(v3, 21);
v3 = Long.rotateLeft(v3, 21);
v3 ^= v0;
v2 += v1;
v1 = rotateLeft(v1, 17);
v1 = Long.rotateLeft(v1, 17);
v1 ^= v2;
v2 = rotateLeft(v2, 32);
v2 = Long.rotateLeft(v2, 32);
}
}
private static long rotateLeft(long value, int shift) {
return (value << shift) | value >>> (64 - shift);
}
}
public static long hash(byte[] data) {

View File

@@ -58,4 +58,9 @@ public class TableUtils {
}
throw new AconfigStorageException("Number of items in a hash table exceeds limit");
}
public static int getBucketIndex(byte[] val, int numBuckets) {
long hashVal = SipHasher13.hash(val);
return (int) Long.remainderUnsigned(hashVal, numBuckets);
}
}

View File

@@ -30,9 +30,10 @@ android_test {
static_libs: [
"androidx.test.runner",
"junit",
"aconfig_storage_file_java",
],
test_config: "AndroidStorageJaveTest.xml",
certificate: "platform",
sdk_version: "test_current",
data: [
"package.map",
"flag.map",
@@ -42,4 +43,5 @@ android_test {
test_suites: [
"general-tests",
],
jarjar_rules: "jarjar.txt",
}

View File

@@ -0,0 +1,15 @@
rule android.aconfig.storage.AconfigStorageException android.aconfig.storage.test.AconfigStorageException
rule android.aconfig.storage.FlagTable android.aconfig.storage.test.FlagTable
rule android.aconfig.storage.PackageTable android.aconfig.storage.test.PackageTable
rule android.aconfig.storage.ByteBufferReader android.aconfig.storage.test.ByteBufferReader
rule android.aconfig.storage.FlagType android.aconfig.storage.test.FlagType
rule android.aconfig.storage.SipHasher13 android.aconfig.storage.test.SipHasher13
rule android.aconfig.storage.FileType android.aconfig.storage.test.FileType
rule android.aconfig.storage.FlagValueList android.aconfig.storage.test.FlagValueList
rule android.aconfig.storage.TableUtils android.aconfig.storage.test.TableUtils
rule android.aconfig.storage.FlagTable$* android.aconfig.storage.test.FlagTable$@1
rule android.aconfig.storage.PackageTable$* android.aconfig.storage.test.PackageTable$@1
rule android.aconfig.storage.FlagValueList$* android.aconfig.storage.test.FlagValueList@1
rule android.aconfig.storage.SipHasher13$* android.aconfig.storage.test.SipHasher13@1

View File

@@ -147,6 +147,7 @@ rust_ffi_shared {
crate_name: "aconfig_storage_read_api_rust_jni",
srcs: ["srcs/lib.rs"],
rustlibs: [
"libaconfig_storage_file",
"libaconfig_storage_read_api",
"libanyhow",
"libjni",

View File

@@ -16,18 +16,14 @@
package android.aconfig.storage;
import dalvik.annotation.optimization.FastNative;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import android.aconfig.storage.PackageReadContext;
import android.aconfig.storage.FlagReadContext;
import dalvik.annotation.optimization.FastNative;
public class AconfigStorageReadAPI {
@@ -50,9 +46,8 @@ public class AconfigStorageReadAPI {
}
// Map a storage file given container and file type
public static MappedByteBuffer getMappedFile(
String container,
StorageFileType type) throws IOException{
public static MappedByteBuffer getMappedFile(String container, StorageFileType type)
throws IOException {
switch (type) {
case PACKAGE_MAP:
return mapStorageFile(STORAGEDIR + "/maps/" + container + ".package.map");
@@ -73,14 +68,14 @@ public class AconfigStorageReadAPI {
// @throws IOException if the passed in file is not a valid package map file
@FastNative
private static native ByteBuffer getPackageReadContextImpl(
ByteBuffer mappedFile, String packageName) throws IOException;
ByteBuffer mappedFile, String packageName) throws IOException;
// API to get package read context
// @param mappedFile: memory mapped package map file
// @param packageName: package name
// @throws IOException if the passed in file is not a valid package map file
static public PackageReadContext getPackageReadContext (
ByteBuffer mappedFile, String packageName) throws IOException {
public static PackageReadContext getPackageReadContext(
ByteBuffer mappedFile, String packageName) throws IOException {
ByteBuffer buffer = getPackageReadContextImpl(mappedFile, packageName);
buffer.order(ByteOrder.LITTLE_ENDIAN);
return new PackageReadContext(buffer.getInt(), buffer.getInt(4));
@@ -94,7 +89,7 @@ public class AconfigStorageReadAPI {
// @throws IOException if the passed in file is not a valid flag map file
@FastNative
private static native ByteBuffer getFlagReadContextImpl(
ByteBuffer mappedFile, int packageId, String flagName) throws IOException;
ByteBuffer mappedFile, int packageId, String flagName) throws IOException;
// API to get flag read context
// @param mappedFile: memory mapped flag map file
@@ -103,7 +98,7 @@ public class AconfigStorageReadAPI {
// @param flagName: flag name
// @throws IOException if the passed in file is not a valid flag map file
public static FlagReadContext getFlagReadContext(
ByteBuffer mappedFile, int packageId, String flagName) throws IOException {
ByteBuffer mappedFile, int packageId, String flagName) throws IOException {
ByteBuffer buffer = getFlagReadContextImpl(mappedFile, packageId, flagName);
buffer.order(ByteOrder.LITTLE_ENDIAN);
return new FlagReadContext(buffer.getInt(), buffer.getInt(4));
@@ -115,8 +110,11 @@ public class AconfigStorageReadAPI {
// @throws IOException if the passed in file is not a valid flag value file or the
// flag index went over the file boundary.
@FastNative
public static native boolean getBooleanFlagValue(
ByteBuffer mappedFile, int flagIndex) throws IOException;
public static native boolean getBooleanFlagValue(ByteBuffer mappedFile, int flagIndex)
throws IOException;
@FastNative
public static native long hash(String packageName) throws IOException;
static {
System.loadLibrary("aconfig_storage_read_api_rust_jni");

View File

@@ -1,5 +1,6 @@
//! aconfig storage read api java rust interlop
use aconfig_storage_file::SipHasher13;
use aconfig_storage_read_api::flag_table_query::find_flag_read_context;
use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value;
use aconfig_storage_read_api::package_table_query::find_package_read_context;
@@ -7,8 +8,9 @@ use aconfig_storage_read_api::{FlagReadContext, PackageReadContext};
use anyhow::Result;
use jni::objects::{JByteBuffer, JClass, JString};
use jni::sys::{jboolean, jint};
use jni::sys::{jboolean, jint, jlong};
use jni::JNIEnv;
use std::hash::Hasher;
/// Call rust find package read context
fn get_package_read_context_java(
@@ -158,3 +160,30 @@ pub extern "system" fn Java_android_aconfig_storage_AconfigStorageReadAPI_getBoo
}
}
}
/// Get flag value JNI
#[no_mangle]
#[allow(unused)]
pub extern "system" fn Java_android_aconfig_storage_AconfigStorageReadAPI_hash<'local>(
mut env: JNIEnv<'local>,
class: JClass<'local>,
package_name: JString<'local>,
) -> jlong {
match siphasher13_hash(&mut env, package_name) {
Ok(value) => value as jlong,
Err(errmsg) => {
env.throw(("java/io/IOException", errmsg.to_string())).expect("failed to throw");
0i64
}
}
}
fn siphasher13_hash(env: &mut JNIEnv, package_name: JString) -> Result<u64> {
// SAFETY:
// The safety here is ensured as the flag name is guaranteed to be a java string
let flag_name: String = unsafe { env.get_string_unchecked(&package_name)?.into() };
let mut s = SipHasher13::new();
s.write(flag_name.as_bytes());
s.write_u8(0xff);
Ok(s.finish())
}

View File

@@ -16,28 +16,29 @@
package android.aconfig.storage.test;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import android.aconfig.DeviceProtos;
import android.aconfig.nano.Aconfig.parsed_flag;
import android.aconfig.storage.AconfigStorageReadAPI;
import android.aconfig.storage.FlagReadContext;
import android.aconfig.storage.FlagReadContext.StoredFlagType;
import android.aconfig.storage.PackageReadContext;
import android.aconfig.storage.SipHasher13;
import android.aconfig.storage.StorageInternalReader;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import android.aconfig.storage.AconfigStorageReadAPI;
import android.aconfig.storage.PackageReadContext;
import android.aconfig.storage.FlagReadContext;
import android.aconfig.storage.FlagReadContext.StoredFlagType;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.util.ArrayList;
import java.util.List;
@RunWith(JUnit4.class)
public class AconfigStorageReadAPITest{
public class AconfigStorageReadAPITest {
private String mStorageDir = "/data/local/tmp/aconfig_java_api_test";
@@ -45,26 +46,29 @@ public class AconfigStorageReadAPITest{
public void testPackageContextQuery() {
MappedByteBuffer packageMap = null;
try {
packageMap = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/maps/mockup.package.map");
} catch(IOException ex){
packageMap =
AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(packageMap != null);
try {
PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_1");
PackageReadContext context =
AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_1");
assertEquals(context.mPackageId, 0);
assertEquals(context.mBooleanStartIndex, 0);
context = AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_2");
context =
AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_2");
assertEquals(context.mPackageId, 1);
assertEquals(context.mBooleanStartIndex, 3);
context = AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_4");
context =
AconfigStorageReadAPI.getPackageReadContext(
packageMap, "com.android.aconfig.storage.test_4");
assertEquals(context.mPackageId, 2);
assertEquals(context.mBooleanStartIndex, 6);
} catch (IOException ex) {
@@ -76,19 +80,19 @@ public class AconfigStorageReadAPITest{
public void testNonExistPackageContextQuery() {
MappedByteBuffer packageMap = null;
try {
packageMap = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/maps/mockup.package.map");
} catch(IOException ex){
packageMap =
AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(packageMap != null);
try {
PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
packageMap, "unknown");
PackageReadContext context =
AconfigStorageReadAPI.getPackageReadContext(packageMap, "unknown");
assertEquals(context.mPackageId, -1);
assertEquals(context.mBooleanStartIndex, -1);
} catch(IOException ex){
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
}
@@ -97,12 +101,11 @@ public class AconfigStorageReadAPITest{
public void testFlagContextQuery() {
MappedByteBuffer flagMap = null;
try {
flagMap = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/maps/mockup.flag.map");
} catch(IOException ex){
flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(flagMap!= null);
assertTrue(flagMap != null);
class Baseline {
public int mPackageId;
@@ -110,10 +113,8 @@ public class AconfigStorageReadAPITest{
public StoredFlagType mFlagType;
public int mFlagIndex;
public Baseline(int packageId,
String flagName,
StoredFlagType flagType,
int flagIndex) {
public Baseline(
int packageId, String flagName, StoredFlagType flagType, int flagIndex) {
mPackageId = packageId;
mFlagName = flagName;
mFlagType = flagType;
@@ -133,8 +134,9 @@ public class AconfigStorageReadAPITest{
try {
for (Baseline baseline : baselines) {
FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
flagMap, baseline.mPackageId, baseline.mFlagName);
FlagReadContext context =
AconfigStorageReadAPI.getFlagReadContext(
flagMap, baseline.mPackageId, baseline.mFlagName);
assertEquals(context.mFlagType, baseline.mFlagType);
assertEquals(context.mFlagIndex, baseline.mFlagIndex);
}
@@ -147,21 +149,19 @@ public class AconfigStorageReadAPITest{
public void testNonExistFlagContextQuery() {
MappedByteBuffer flagMap = null;
try {
flagMap = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/maps/mockup.flag.map");
} catch(IOException ex){
flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(flagMap!= null);
assertTrue(flagMap != null);
try {
FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
flagMap, 0, "unknown");
FlagReadContext context =
AconfigStorageReadAPI.getFlagReadContext(flagMap, 0, "unknown");
assertEquals(context.mFlagType, null);
assertEquals(context.mFlagIndex, -1);
context = AconfigStorageReadAPI.getFlagReadContext(
flagMap, 3, "enabled_ro");
context = AconfigStorageReadAPI.getFlagReadContext(flagMap, 3, "enabled_ro");
assertEquals(context.mFlagType, null);
assertEquals(context.mFlagIndex, -1);
} catch (IOException ex) {
@@ -173,12 +173,11 @@ public class AconfigStorageReadAPITest{
public void testBooleanFlagValueQuery() {
MappedByteBuffer flagVal = null;
try {
flagVal = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/boot/mockup.val");
flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(flagVal!= null);
assertTrue(flagVal != null);
boolean[] baselines = {false, true, true, false, true, true, true, true};
for (int i = 0; i < 8; ++i) {
@@ -195,12 +194,11 @@ public class AconfigStorageReadAPITest{
public void testInvalidBooleanFlagValueQuery() {
MappedByteBuffer flagVal = null;
try {
flagVal = AconfigStorageReadAPI.mapStorageFile(
mStorageDir + "/boot/mockup.val");
flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(flagVal!= null);
assertTrue(flagVal != null);
try {
Boolean value = AconfigStorageReadAPI.getBooleanFlagValue(flagVal, 9);
@@ -210,4 +208,63 @@ public class AconfigStorageReadAPITest{
assertTrue(ex.toString(), ex.toString().contains(expectedErrmsg));
}
}
}
@Test
public void testRustJavaEqualHash() throws IOException {
List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
for (parsed_flag flag : flags) {
String packageName = flag.package_;
String flagName = flag.name;
long rHash = AconfigStorageReadAPI.hash(packageName);
long jHash = SipHasher13.hash(packageName.getBytes());
assertEquals(rHash, jHash);
String fullFlagName = packageName + "/" + flagName;
rHash = AconfigStorageReadAPI.hash(fullFlagName);
jHash = SipHasher13.hash(fullFlagName.getBytes());
assertEquals(rHash, jHash);
}
}
@Test
public void testRustJavaEqualFlag() throws IOException {
List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
String mapPath = "/metadata/aconfig/maps/";
String flagsPath = "/metadata/aconfig/boot/";
for (parsed_flag flag : flags) {
String container = flag.container;
String packageName = flag.package_;
String flagName = flag.name;
String fullFlagName = packageName + "/" + flagName;
MappedByteBuffer packageMap =
AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".package.map");
MappedByteBuffer flagMap =
AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".flag.map");
MappedByteBuffer flagValList =
AconfigStorageReadAPI.mapStorageFile(flagsPath + container + ".val");
PackageReadContext packageContext =
AconfigStorageReadAPI.getPackageReadContext(packageMap, packageName);
FlagReadContext flagContext =
AconfigStorageReadAPI.getFlagReadContext(
flagMap, packageContext.mPackageId, flagName);
boolean rVal =
AconfigStorageReadAPI.getBooleanFlagValue(
flagValList,
packageContext.mBooleanStartIndex + flagContext.mFlagIndex);
StorageInternalReader reader = new StorageInternalReader(container, packageName);
boolean jVal = reader.getBooleanFlagValue(flagContext.mFlagIndex);
long rHash = AconfigStorageReadAPI.hash(packageName);
long jHash = SipHasher13.hash(packageName.getBytes());
assertEquals(rVal, jVal);
}
}
}

View File

@@ -2,6 +2,8 @@ android_test {
name: "aconfig_storage_read_api.test.java",
srcs: ["./**/*.java"],
static_libs: [
"aconfig_device_paths_java",
"aconfig_storage_file_java",
"aconfig_storage_reader_java",
"androidx.test.rules",
"libaconfig_storage_read_api_java",