diff --git a/tools/aconfig/Cargo.toml b/tools/aconfig/Cargo.toml index 8d93261934..535ff214ed 100644 --- a/tools/aconfig/Cargo.toml +++ b/tools/aconfig/Cargo.toml @@ -3,5 +3,6 @@ members = [ "aconfig", "aconfig_protos", + "aconfig_storage_file", "printflags" ] diff --git a/tools/aconfig/aconfig/Android.bp b/tools/aconfig/aconfig/Android.bp index 3be456c97f..3152d350d4 100644 --- a/tools/aconfig/aconfig/Android.bp +++ b/tools/aconfig/aconfig/Android.bp @@ -10,6 +10,7 @@ rust_defaults { srcs: ["src/main.rs"], rustlibs: [ "libaconfig_protos", + "libaconfig_storage_file", "libanyhow", "libclap", "libitertools", diff --git a/tools/aconfig/aconfig/Cargo.toml b/tools/aconfig/aconfig/Cargo.toml index 01ad8c6760..abd3ee01e8 100644 --- a/tools/aconfig/aconfig/Cargo.toml +++ b/tools/aconfig/aconfig/Cargo.toml @@ -16,3 +16,4 @@ serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" tinytemplate = "1.2.1" aconfig_protos = { path = "../aconfig_protos" } +aconfig_storage_file = { path = "../aconfig_storage_file" } diff --git a/tools/aconfig/aconfig/src/codegen/mod.rs b/tools/aconfig/aconfig/src/codegen/mod.rs index 7b2336f987..1ea3b37849 100644 --- a/tools/aconfig/aconfig/src/codegen/mod.rs +++ b/tools/aconfig/aconfig/src/codegen/mod.rs @@ -18,9 +18,9 @@ pub mod cpp; pub mod java; pub mod rust; +use aconfig_protos::{is_valid_name_ident, is_valid_package_ident}; use anyhow::{ensure, Result}; use clap::ValueEnum; -use aconfig_protos::{is_valid_name_ident, is_valid_package_ident}; pub fn create_device_config_ident(package: &str, flag_name: &str) -> Result { ensure!(is_valid_package_ident(package), "bad package"); diff --git a/tools/aconfig/aconfig/src/commands.rs b/tools/aconfig/aconfig/src/commands.rs index 93bc436fd8..59f349b35d 100644 --- a/tools/aconfig/aconfig/src/commands.rs +++ b/tools/aconfig/aconfig/src/commands.rs @@ -26,12 +26,12 @@ use crate::codegen::java::generate_java_code; use crate::codegen::rust::generate_rust_code; use crate::codegen::CodegenMode; use crate::dump::{DumpFormat, DumpPredicate}; +use crate::storage::generate_storage_file; use aconfig_protos::{ ParsedFlagExt, ProtoFlagMetadata, ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag, ProtoParsedFlags, ProtoTracepoint, }; -use crate::storage::generate_storage_file; -use crate::storage::StorageFileSelection; +use aconfig_storage_file::StorageFileSelection; pub struct Input { pub source: String, @@ -224,7 +224,11 @@ pub fn create_rust_lib(mut input: Input, codegen_mode: CodegenMode) -> Result, container: &str, file: &StorageFileSelection) -> Result> { +pub fn create_storage( + caches: Vec, + container: &str, + file: &StorageFileSelection, +) -> Result> { let parsed_flags_vec: Vec = caches .into_iter() .map(|mut input| input.try_parse_flags()) diff --git a/tools/aconfig/aconfig/src/dump.rs b/tools/aconfig/aconfig/src/dump.rs index 12352f99f8..2a29c2bccf 100644 --- a/tools/aconfig/aconfig/src/dump.rs +++ b/tools/aconfig/aconfig/src/dump.rs @@ -197,8 +197,8 @@ fn create_filter_predicate_single(filter: &str) -> Result> { #[cfg(test)] mod tests { use super::*; - use aconfig_protos::ProtoParsedFlags; use crate::test::parse_test_flags; + use aconfig_protos::ProtoParsedFlags; use protobuf::Message; fn parse_enabled_ro_flag() -> ProtoParsedFlag { diff --git a/tools/aconfig/aconfig/src/main.rs b/tools/aconfig/aconfig/src/main.rs index 30a7e9db92..5a4f23c4d3 100644 --- a/tools/aconfig/aconfig/src/main.rs +++ b/tools/aconfig/aconfig/src/main.rs @@ -29,9 +29,9 @@ mod commands; mod dump; mod storage; +use aconfig_storage_file::StorageFileSelection; use codegen::CodegenMode; use dump::DumpFormat; -use storage::StorageFileSelection; #[cfg(test)] mod test; @@ -213,8 +213,10 @@ fn main() -> Result<()> { get_optional_arg::(sub_matches, "container").map(|c| c.as_str()); let declarations = open_zero_or_more_files(sub_matches, "declarations")?; let values = open_zero_or_more_files(sub_matches, "values")?; - let default_permission = - get_required_arg::(sub_matches, "default-permission")?; + let default_permission = get_required_arg::( + sub_matches, + "default-permission", + )?; let output = commands::parse_flags( package, container, diff --git a/tools/aconfig/aconfig/src/storage/flag_table.rs b/tools/aconfig/aconfig/src/storage/flag_table.rs index 3545700b6a..70878a8b58 100644 --- a/tools/aconfig/aconfig/src/storage/flag_table.rs +++ b/tools/aconfig/aconfig/src/storage/flag_table.rs @@ -15,267 +15,134 @@ */ use crate::commands::assign_flag_ids; -use crate::storage::{self, FlagPackage}; +use crate::storage::FlagPackage; +use aconfig_storage_file::{ + get_bucket_index, get_table_size, FlagTable, FlagTableHeader, FlagTableNode, FILE_VERSION, +}; use anyhow::{anyhow, Result}; -#[derive(PartialEq, Debug)] -pub struct FlagTableHeader { - pub version: u32, - pub container: String, - pub file_size: u32, - pub num_flags: u32, - pub bucket_offset: u32, - pub node_offset: u32, -} - -impl FlagTableHeader { - fn new(container: &str, num_flags: u32) -> Self { - Self { - version: storage::FILE_VERSION, - container: String::from(container), - file_size: 0, - num_flags, - bucket_offset: 0, - node_offset: 0, - } - } - - fn as_bytes(&self) -> Vec { - let mut result = Vec::new(); - result.extend_from_slice(&self.version.to_le_bytes()); - let container_bytes = self.container.as_bytes(); - result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); - result.extend_from_slice(container_bytes); - result.extend_from_slice(&self.file_size.to_le_bytes()); - result.extend_from_slice(&self.num_flags.to_le_bytes()); - result.extend_from_slice(&self.bucket_offset.to_le_bytes()); - result.extend_from_slice(&self.node_offset.to_le_bytes()); - result +fn new_header(container: &str, num_flags: u32) -> FlagTableHeader { + FlagTableHeader { + version: FILE_VERSION, + container: String::from(container), + file_size: 0, + num_flags, + bucket_offset: 0, + node_offset: 0, } } -#[derive(PartialEq, Debug, Clone)] -pub struct FlagTableNode { - pub package_id: u32, - pub flag_name: String, - pub flag_type: u16, - pub flag_id: u16, - pub next_offset: Option, - pub bucket_index: u32, -} - -impl FlagTableNode { - fn new( - package_id: u32, - flag_name: &str, - flag_type: u16, - flag_id: u16, - num_buckets: u32, - ) -> Self { - let full_flag_name = package_id.to_string() + "/" + flag_name; - let bucket_index = storage::get_bucket_index(&full_flag_name, num_buckets); - Self { - package_id, - flag_name: flag_name.to_string(), - flag_type, - flag_id, - next_offset: None, - bucket_index, - } - } - - fn as_bytes(&self) -> Vec { - let mut result = Vec::new(); - result.extend_from_slice(&self.package_id.to_le_bytes()); - let name_bytes = self.flag_name.as_bytes(); - result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); - result.extend_from_slice(name_bytes); - result.extend_from_slice(&self.flag_type.to_le_bytes()); - result.extend_from_slice(&self.flag_id.to_le_bytes()); - result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); - result +fn new_node( + package_id: u32, + flag_name: &str, + flag_type: u16, + flag_id: u16, + num_buckets: u32, +) -> FlagTableNode { + let full_flag_name = package_id.to_string() + "/" + flag_name; + let bucket_index = get_bucket_index(&full_flag_name, num_buckets); + FlagTableNode { + package_id, + flag_name: flag_name.to_string(), + flag_type, + flag_id, + next_offset: None, + bucket_index, } } -#[derive(PartialEq, Debug)] -pub struct FlagTable { - pub header: FlagTableHeader, - pub buckets: Vec>, - pub nodes: Vec, +fn create_nodes(package: &FlagPackage, num_buckets: u32) -> Result> { + let flag_ids = assign_flag_ids(package.package_name, package.boolean_flags.iter().copied())?; + package + .boolean_flags + .iter() + .map(|&pf| { + let fid = flag_ids + .get(pf.name()) + .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; + // all flags are boolean value at the moment, thus using the last bit. When more + // flag value types are supported, flag value type information should come from the + // parsed flag, and we will set the flag_type bit mask properly. + let flag_type = 1; + Ok(new_node(package.package_id, pf.name(), flag_type, *fid, num_buckets)) + }) + .collect::>>() } -impl FlagTable { - fn create_nodes(package: &FlagPackage, num_buckets: u32) -> Result> { - let flag_ids = - assign_flag_ids(package.package_name, package.boolean_flags.iter().copied())?; - package - .boolean_flags +pub fn create_flag_table(container: &str, packages: &[FlagPackage]) -> Result { + // create table + let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); + let num_buckets = get_table_size(num_flags)?; + + let mut table = FlagTable { + header: new_header(container, num_flags), + buckets: vec![None; num_buckets as usize], + nodes: packages .iter() - .map(|&pf| { - let fid = flag_ids - .get(pf.name()) - .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; - // all flags are boolean value at the moment, thus using the last bit. When more - // flag value types are supported, flag value type information should come from the - // parsed flag, and we will set the flag_type bit mask properly. - let flag_type = 1; - Ok(FlagTableNode::new(package.package_id, pf.name(), flag_type, *fid, num_buckets)) - }) - .collect::>>() - } + .map(|pkg| create_nodes(pkg, num_buckets)) + .collect::>>()? + .concat(), + }; - pub fn new(container: &str, packages: &[FlagPackage]) -> Result { - // create table - let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); - let num_buckets = storage::get_table_size(num_flags)?; + // initialize all header fields + table.header.bucket_offset = table.header.as_bytes().len() as u32; + table.header.node_offset = table.header.bucket_offset + num_buckets * 4; + table.header.file_size = table.header.node_offset + + table.nodes.iter().map(|x| x.as_bytes().len()).sum::() as u32; - let mut table = Self { - header: FlagTableHeader::new(container, num_flags), - buckets: vec![None; num_buckets as usize], - nodes: packages - .iter() - .map(|pkg| FlagTable::create_nodes(pkg, num_buckets)) - .collect::>>()? - .concat(), - }; + // sort nodes by bucket index for efficiency + table.nodes.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); - // initialize all header fields - table.header.bucket_offset = table.header.as_bytes().len() as u32; - table.header.node_offset = table.header.bucket_offset + num_buckets * 4; - table.header.file_size = table.header.node_offset - + table.nodes.iter().map(|x| x.as_bytes().len()).sum::() as u32; + // fill all node offset + let mut offset = table.header.node_offset; + for i in 0..table.nodes.len() { + let node_bucket_idx = table.nodes[i].bucket_index; + let next_node_bucket_idx = + if i + 1 < table.nodes.len() { Some(table.nodes[i + 1].bucket_index) } else { None }; - // sort nodes by bucket index for efficiency - table.nodes.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); + if table.buckets[node_bucket_idx as usize].is_none() { + table.buckets[node_bucket_idx as usize] = Some(offset); + } + offset += table.nodes[i].as_bytes().len() as u32; - // fill all node offset - let mut offset = table.header.node_offset; - for i in 0..table.nodes.len() { - let node_bucket_idx = table.nodes[i].bucket_index; - let next_node_bucket_idx = if i + 1 < table.nodes.len() { - Some(table.nodes[i + 1].bucket_index) - } else { - None - }; - - if table.buckets[node_bucket_idx as usize].is_none() { - table.buckets[node_bucket_idx as usize] = Some(offset); - } - offset += table.nodes[i].as_bytes().len() as u32; - - if let Some(index) = next_node_bucket_idx { - if index == node_bucket_idx { - table.nodes[i].next_offset = Some(offset); - } + if let Some(index) = next_node_bucket_idx { + if index == node_bucket_idx { + table.nodes[i].next_offset = Some(offset); } } - - Ok(table) } - pub fn as_bytes(&self) -> Vec { - [ - self.header.as_bytes(), - self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::>().concat(), - self.nodes.iter().map(|v| v.as_bytes()).collect::>().concat(), - ] - .concat() - } + Ok(table) } #[cfg(test)] mod tests { use super::*; - use crate::storage::{ - group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes, - tests::read_u16_from_bytes, tests::read_u32_from_bytes, - }; + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; - impl FlagTableHeader { - // test only method to deserialize back into the header struct - fn from_bytes(bytes: &[u8]) -> Result { - let mut head = 0; - Ok(Self { - version: read_u32_from_bytes(bytes, &mut head)?, - container: read_str_from_bytes(bytes, &mut head)?, - file_size: read_u32_from_bytes(bytes, &mut head)?, - num_flags: read_u32_from_bytes(bytes, &mut head)?, - bucket_offset: read_u32_from_bytes(bytes, &mut head)?, - node_offset: read_u32_from_bytes(bytes, &mut head)?, - }) + // create test baseline, syntactic sugar + fn new_expected_node( + package_id: u32, + flag_name: &str, + flag_type: u16, + flag_id: u16, + next_offset: Option, + bucket_index: u32, + ) -> FlagTableNode { + FlagTableNode { + package_id, + flag_name: flag_name.to_string(), + flag_type, + flag_id, + next_offset, + bucket_index, } } - impl FlagTableNode { - // test only method to deserialize back into the node struct - fn from_bytes(bytes: &[u8], num_buckets: u32) -> Result { - let mut head = 0; - let mut node = Self { - package_id: read_u32_from_bytes(bytes, &mut head)?, - flag_name: read_str_from_bytes(bytes, &mut head)?, - flag_type: read_u16_from_bytes(bytes, &mut head)?, - flag_id: read_u16_from_bytes(bytes, &mut head)?, - next_offset: match read_u32_from_bytes(bytes, &mut head)? { - 0 => None, - val => Some(val), - }, - bucket_index: 0, - }; - let full_flag_name = node.package_id.to_string() + "/" + &node.flag_name; - node.bucket_index = storage::get_bucket_index(&full_flag_name, num_buckets); - Ok(node) - } - - // create test baseline, syntactic sugar - fn new_expected( - package_id: u32, - flag_name: &str, - flag_type: u16, - flag_id: u16, - next_offset: Option, - bucket_index: u32, - ) -> Self { - Self { - package_id, - flag_name: flag_name.to_string(), - flag_type, - flag_id, - next_offset, - bucket_index, - } - } - } - - impl FlagTable { - // test only method to deserialize back into the table struct - fn from_bytes(bytes: &[u8]) -> Result { - let header = FlagTableHeader::from_bytes(bytes)?; - let num_flags = header.num_flags; - let num_buckets = storage::get_table_size(num_flags)?; - let mut head = header.as_bytes().len(); - let buckets = (0..num_buckets) - .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { - 0 => None, - val => Some(val), - }) - .collect(); - let nodes = (0..num_flags) - .map(|_| { - let node = FlagTableNode::from_bytes(&bytes[head..], num_buckets).unwrap(); - head += node.as_bytes().len(); - node - }) - .collect(); - - let table = Self { header, buckets, nodes }; - Ok(table) - } - } - - pub fn create_test_flag_table() -> Result { + fn create_test_flag_table() -> Result { let caches = parse_all_test_flags(); let packages = group_flags_by_package(caches.iter()); - FlagTable::new("system", &packages) + create_flag_table("system", &packages) } #[test] @@ -286,7 +153,7 @@ mod tests { let header: &FlagTableHeader = &flag_table.as_ref().unwrap().header; let expected_header = FlagTableHeader { - version: storage::FILE_VERSION, + version: FILE_VERSION, container: String::from("system"), file_size: 320, num_flags: 8, @@ -320,39 +187,13 @@ mod tests { let nodes: &Vec = &flag_table.as_ref().unwrap().nodes; assert_eq!(nodes.len(), 8); - assert_eq!(nodes[0], FlagTableNode::new_expected(0, "enabled_ro", 1, 1, None, 0)); - assert_eq!(nodes[1], FlagTableNode::new_expected(0, "enabled_rw", 1, 2, Some(150), 1)); - assert_eq!(nodes[2], FlagTableNode::new_expected(1, "disabled_ro", 1, 0, None, 1)); - assert_eq!(nodes[3], FlagTableNode::new_expected(2, "enabled_ro", 1, 1, None, 5)); - assert_eq!( - nodes[4], - FlagTableNode::new_expected(1, "enabled_fixed_ro", 1, 1, Some(235), 7) - ); - assert_eq!(nodes[5], FlagTableNode::new_expected(1, "enabled_ro", 1, 2, None, 7)); - assert_eq!(nodes[6], FlagTableNode::new_expected(2, "enabled_fixed_ro", 1, 0, None, 9)); - assert_eq!(nodes[7], FlagTableNode::new_expected(0, "disabled_rw", 1, 0, None, 15)); - } - - #[test] - // this test point locks down the table serialization - fn test_serialization() { - let flag_table = create_test_flag_table().unwrap(); - - let header: &FlagTableHeader = &flag_table.header; - let reinterpreted_header = FlagTableHeader::from_bytes(&header.as_bytes()); - assert!(reinterpreted_header.is_ok()); - assert_eq!(header, &reinterpreted_header.unwrap()); - - let nodes: &Vec = &flag_table.nodes; - let num_buckets = storage::get_table_size(header.num_flags).unwrap(); - for node in nodes.iter() { - let reinterpreted_node = FlagTableNode::from_bytes(&node.as_bytes(), num_buckets); - assert!(reinterpreted_node.is_ok()); - assert_eq!(node, &reinterpreted_node.unwrap()); - } - - let reinterpreted_table = FlagTable::from_bytes(&flag_table.as_bytes()); - assert!(reinterpreted_table.is_ok()); - assert_eq!(&flag_table, &reinterpreted_table.unwrap()); + assert_eq!(nodes[0], new_expected_node(0, "enabled_ro", 1, 1, None, 0)); + assert_eq!(nodes[1], new_expected_node(0, "enabled_rw", 1, 2, Some(150), 1)); + assert_eq!(nodes[2], new_expected_node(1, "disabled_ro", 1, 0, None, 1)); + assert_eq!(nodes[3], new_expected_node(2, "enabled_ro", 1, 1, None, 5)); + assert_eq!(nodes[4], new_expected_node(1, "enabled_fixed_ro", 1, 1, Some(235), 7)); + assert_eq!(nodes[5], new_expected_node(1, "enabled_ro", 1, 2, None, 7)); + assert_eq!(nodes[6], new_expected_node(2, "enabled_fixed_ro", 1, 0, None, 9)); + assert_eq!(nodes[7], new_expected_node(0, "disabled_rw", 1, 0, None, 15)); } } diff --git a/tools/aconfig/aconfig/src/storage/flag_value.rs b/tools/aconfig/aconfig/src/storage/flag_value.rs index 3c5bb17384..0d4b5b480e 100644 --- a/tools/aconfig/aconfig/src/storage/flag_value.rs +++ b/tools/aconfig/aconfig/src/storage/flag_value.rs @@ -15,132 +15,58 @@ */ use crate::commands::assign_flag_ids; +use crate::storage::FlagPackage; use aconfig_protos::ProtoFlagState; -use crate::storage::{self, FlagPackage}; +use aconfig_storage_file::{FlagValueHeader, FlagValueList, FILE_VERSION}; use anyhow::{anyhow, Result}; -#[derive(PartialEq, Debug)] -pub struct FlagValueHeader { - pub version: u32, - pub container: String, - pub file_size: u32, - pub num_flags: u32, - pub boolean_value_offset: u32, +fn new_header(container: &str, num_flags: u32) -> FlagValueHeader { + FlagValueHeader { + version: FILE_VERSION, + container: String::from(container), + file_size: 0, + num_flags, + boolean_value_offset: 0, + } } -impl FlagValueHeader { - fn new(container: &str, num_flags: u32) -> Self { - Self { - version: storage::FILE_VERSION, - container: String::from(container), - file_size: 0, - num_flags, - boolean_value_offset: 0, +pub fn create_flag_value(container: &str, packages: &[FlagPackage]) -> Result { + // create list + let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); + + let mut list = FlagValueList { + header: new_header(container, num_flags), + booleans: vec![false; num_flags as usize], + }; + + for pkg in packages.iter() { + let start_offset = pkg.boolean_offset as usize; + let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?; + for pf in pkg.boolean_flags.iter() { + let fid = flag_ids + .get(pf.name()) + .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; + + list.booleans[start_offset + (*fid as usize)] = pf.state() == ProtoFlagState::ENABLED; } } - fn as_bytes(&self) -> Vec { - let mut result = Vec::new(); - result.extend_from_slice(&self.version.to_le_bytes()); - let container_bytes = self.container.as_bytes(); - result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); - result.extend_from_slice(container_bytes); - result.extend_from_slice(&self.file_size.to_le_bytes()); - result.extend_from_slice(&self.num_flags.to_le_bytes()); - result.extend_from_slice(&self.boolean_value_offset.to_le_bytes()); - result - } -} + // initialize all header fields + list.header.boolean_value_offset = list.header.as_bytes().len() as u32; + list.header.file_size = list.header.boolean_value_offset + num_flags; -#[derive(PartialEq, Debug)] -pub struct FlagValueList { - pub header: FlagValueHeader, - pub booleans: Vec, -} - -impl FlagValueList { - pub fn new(container: &str, packages: &[FlagPackage]) -> Result { - // create list - let num_flags = packages.iter().map(|pkg| pkg.boolean_flags.len() as u32).sum(); - - let mut list = Self { - header: FlagValueHeader::new(container, num_flags), - booleans: vec![false; num_flags as usize], - }; - - for pkg in packages.iter() { - let start_offset = pkg.boolean_offset as usize; - let flag_ids = assign_flag_ids(pkg.package_name, pkg.boolean_flags.iter().copied())?; - for pf in pkg.boolean_flags.iter() { - let fid = flag_ids - .get(pf.name()) - .ok_or(anyhow!(format!("missing flag id for {}", pf.name())))?; - - list.booleans[start_offset + (*fid as usize)] = - pf.state() == ProtoFlagState::ENABLED; - } - } - - // initialize all header fields - list.header.boolean_value_offset = list.header.as_bytes().len() as u32; - list.header.file_size = list.header.boolean_value_offset + num_flags; - - Ok(list) - } - - pub fn as_bytes(&self) -> Vec { - [ - self.header.as_bytes(), - self.booleans - .iter() - .map(|&v| u8::from(v).to_le_bytes()) - .collect::>() - .concat(), - ] - .concat() - } + Ok(list) } #[cfg(test)] mod tests { use super::*; - use crate::storage::{ - group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes, - tests::read_u32_from_bytes, tests::read_u8_from_bytes, - }; - - impl FlagValueHeader { - // test only method to deserialize back into the header struct - fn from_bytes(bytes: &[u8]) -> Result { - let mut head = 0; - Ok(Self { - version: read_u32_from_bytes(bytes, &mut head)?, - container: read_str_from_bytes(bytes, &mut head)?, - file_size: read_u32_from_bytes(bytes, &mut head)?, - num_flags: read_u32_from_bytes(bytes, &mut head)?, - boolean_value_offset: read_u32_from_bytes(bytes, &mut head)?, - }) - } - } - - impl FlagValueList { - // test only method to deserialize back into the flag value struct - fn from_bytes(bytes: &[u8]) -> Result { - let header = FlagValueHeader::from_bytes(bytes)?; - let num_flags = header.num_flags; - let mut head = header.as_bytes().len(); - let booleans = (0..num_flags) - .map(|_| read_u8_from_bytes(bytes, &mut head).unwrap() == 1) - .collect(); - let list = Self { header, booleans }; - Ok(list) - } - } + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; pub fn create_test_flag_value_list() -> Result { let caches = parse_all_test_flags(); let packages = group_flags_by_package(caches.iter()); - FlagValueList::new("system", &packages) + create_flag_value("system", &packages) } #[test] @@ -151,7 +77,7 @@ mod tests { let header: &FlagValueHeader = &flag_value_list.as_ref().unwrap().header; let expected_header = FlagValueHeader { - version: storage::FILE_VERSION, + version: FILE_VERSION, container: String::from("system"), file_size: 34, num_flags: 8, @@ -163,19 +89,4 @@ mod tests { let expected_booleans: Vec = vec![false; header.num_flags as usize]; assert_eq!(booleans, &expected_booleans); } - - #[test] - // this test point locks down the value list serialization - fn test_serialization() { - let flag_value_list = create_test_flag_value_list().unwrap(); - - let header: &FlagValueHeader = &flag_value_list.header; - let reinterpreted_header = FlagValueHeader::from_bytes(&header.as_bytes()); - assert!(reinterpreted_header.is_ok()); - assert_eq!(header, &reinterpreted_header.unwrap()); - - let reinterpreted_value_list = FlagValueList::from_bytes(&flag_value_list.as_bytes()); - assert!(reinterpreted_value_list.is_ok()); - assert_eq!(&flag_value_list, &reinterpreted_value_list.unwrap()); - } } diff --git a/tools/aconfig/aconfig/src/storage/mod.rs b/tools/aconfig/aconfig/src/storage/mod.rs index 4f2dc81a03..29eb9c8093 100644 --- a/tools/aconfig/aconfig/src/storage/mod.rs +++ b/tools/aconfig/aconfig/src/storage/mod.rs @@ -18,59 +18,15 @@ pub mod flag_table; pub mod flag_value; pub mod package_table; -use anyhow::{anyhow, Result}; -use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; +use anyhow::Result; +use std::collections::{HashMap, HashSet}; -use aconfig_protos::{ProtoParsedFlag, ProtoParsedFlags}; use crate::storage::{ - flag_table::FlagTable, flag_value::FlagValueList, package_table::PackageTable, + flag_table::create_flag_table, flag_value::create_flag_value, + package_table::create_package_table, }; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum StorageFileSelection { - PackageMap, - FlagMap, - FlagVal, -} - -impl TryFrom<&str> for StorageFileSelection { - type Error = anyhow::Error; - - fn try_from(value: &str) -> std::result::Result { - match value { - "package_map" => Ok(Self::PackageMap), - "flag_map" => Ok(Self::FlagMap), - "flag_val" => Ok(Self::FlagVal), - _ => Err(anyhow!("Invalid storage file to create")), - } - } -} - -pub const FILE_VERSION: u32 = 1; - -pub const HASH_PRIMES: [u32; 29] = [ - 7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241, - 786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611, - 402653189, 805306457, 1610612741, -]; - -/// Get the right hash table size given number of entries in the table. Use a -/// load factor of 0.5 for performance. -pub fn get_table_size(entries: u32) -> Result { - HASH_PRIMES - .iter() - .find(|&&num| num >= 2 * entries) - .copied() - .ok_or(anyhow!("Number of packages is too large")) -} - -/// Get the corresponding bucket index given the key and number of buckets -pub fn get_bucket_index(val: &T, num_buckets: u32) -> u32 { - let mut s = DefaultHasher::new(); - val.hash(&mut s); - (s.finish() % num_buckets as u64) as u32 -} +use aconfig_protos::{ProtoParsedFlag, ProtoParsedFlags}; +use aconfig_storage_file::StorageFileSelection; pub struct FlagPackage<'a> { pub package_name: &'a str, @@ -140,15 +96,15 @@ where match file { StorageFileSelection::PackageMap => { - let package_table = PackageTable::new(container, &packages)?; + let package_table = create_package_table(container, &packages)?; Ok(package_table.as_bytes()) } StorageFileSelection::FlagMap => { - let flag_table = FlagTable::new(container, &packages)?; + let flag_table = create_flag_table(container, &packages)?; Ok(flag_table.as_bytes()) } StorageFileSelection::FlagVal => { - let flag_value = FlagValueList::new(container, &packages)?; + let flag_value = create_flag_value(container, &packages)?; Ok(flag_value.as_bytes()) } } @@ -159,35 +115,6 @@ mod tests { use super::*; use crate::Input; - /// Read and parse bytes as u8 - pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u8::from_le_bytes(buf[*head..*head + 1].try_into()?); - *head += 1; - Ok(val) - } - - /// Read and parse bytes as u16 - pub fn read_u16_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u16::from_le_bytes(buf[*head..*head + 2].try_into()?); - *head += 2; - Ok(val) - } - - /// Read and parse bytes as u32 - pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u32::from_le_bytes(buf[*head..*head + 4].try_into()?); - *head += 4; - Ok(val) - } - - /// Read and parse bytes as string - pub fn read_str_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let num_bytes = read_u32_from_bytes(buf, head)? as usize; - let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())?; - *head += num_bytes; - Ok(val) - } - pub fn parse_all_test_flags() -> Vec { let aconfig_files = [ ( diff --git a/tools/aconfig/aconfig/src/storage/package_table.rs b/tools/aconfig/aconfig/src/storage/package_table.rs index 40362340e0..f82e932b0e 100644 --- a/tools/aconfig/aconfig/src/storage/package_table.rs +++ b/tools/aconfig/aconfig/src/storage/package_table.rs @@ -14,215 +14,87 @@ * limitations under the License. */ -use crate::storage::{self, FlagPackage}; use anyhow::Result; -#[derive(PartialEq, Debug)] -pub struct PackageTableHeader { - pub version: u32, - pub container: String, - pub file_size: u32, - pub num_packages: u32, - pub bucket_offset: u32, - pub node_offset: u32, +use aconfig_storage_file::{ + get_bucket_index, get_table_size, PackageTable, PackageTableHeader, PackageTableNode, + FILE_VERSION, +}; + +use crate::storage::FlagPackage; + +fn new_header(container: &str, num_packages: u32) -> PackageTableHeader { + PackageTableHeader { + version: FILE_VERSION, + container: String::from(container), + file_size: 0, + num_packages, + bucket_offset: 0, + node_offset: 0, + } } -impl PackageTableHeader { - fn new(container: &str, num_packages: u32) -> Self { - Self { - version: storage::FILE_VERSION, - container: String::from(container), - file_size: 0, - num_packages, - bucket_offset: 0, - node_offset: 0, +fn new_node(package: &FlagPackage, num_buckets: u32) -> PackageTableNode { + let bucket_index = get_bucket_index(&package.package_name.to_string(), num_buckets); + PackageTableNode { + package_name: String::from(package.package_name), + package_id: package.package_id, + boolean_offset: package.boolean_offset, + next_offset: None, + bucket_index, + } +} + +pub fn create_package_table(container: &str, packages: &[FlagPackage]) -> Result { + // create table + let num_packages = packages.len() as u32; + let num_buckets = get_table_size(num_packages)?; + let mut table = PackageTable { + header: new_header(container, num_packages), + buckets: vec![None; num_buckets as usize], + nodes: packages.iter().map(|pkg| new_node(pkg, num_buckets)).collect(), + }; + + // initialize all header fields + table.header.bucket_offset = table.header.as_bytes().len() as u32; + table.header.node_offset = table.header.bucket_offset + num_buckets * 4; + table.header.file_size = table.header.node_offset + + table.nodes.iter().map(|x| x.as_bytes().len()).sum::() as u32; + + // sort nodes by bucket index for efficiency + table.nodes.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); + + // fill all node offset + let mut offset = table.header.node_offset; + for i in 0..table.nodes.len() { + let node_bucket_idx = table.nodes[i].bucket_index; + let next_node_bucket_idx = + if i + 1 < table.nodes.len() { Some(table.nodes[i + 1].bucket_index) } else { None }; + + if table.buckets[node_bucket_idx as usize].is_none() { + table.buckets[node_bucket_idx as usize] = Some(offset); } - } + offset += table.nodes[i].as_bytes().len() as u32; - fn as_bytes(&self) -> Vec { - let mut result = Vec::new(); - result.extend_from_slice(&self.version.to_le_bytes()); - let container_bytes = self.container.as_bytes(); - result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); - result.extend_from_slice(container_bytes); - result.extend_from_slice(&self.file_size.to_le_bytes()); - result.extend_from_slice(&self.num_packages.to_le_bytes()); - result.extend_from_slice(&self.bucket_offset.to_le_bytes()); - result.extend_from_slice(&self.node_offset.to_le_bytes()); - result - } -} - -#[derive(PartialEq, Debug)] -pub struct PackageTableNode { - pub package_name: String, - pub package_id: u32, - // offset of the first boolean flag in this flag package with respect to the start of - // boolean flag value array in the flag value file - pub boolean_offset: u32, - pub next_offset: Option, - pub bucket_index: u32, -} - -impl PackageTableNode { - fn new(package: &FlagPackage, num_buckets: u32) -> Self { - let bucket_index = - storage::get_bucket_index(&package.package_name.to_string(), num_buckets); - Self { - package_name: String::from(package.package_name), - package_id: package.package_id, - boolean_offset: package.boolean_offset, - next_offset: None, - bucket_index, - } - } - - fn as_bytes(&self) -> Vec { - let mut result = Vec::new(); - let name_bytes = self.package_name.as_bytes(); - result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); - result.extend_from_slice(name_bytes); - result.extend_from_slice(&self.package_id.to_le_bytes()); - result.extend_from_slice(&self.boolean_offset.to_le_bytes()); - result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); - result - } -} - -#[derive(PartialEq, Debug)] -pub struct PackageTable { - pub header: PackageTableHeader, - pub buckets: Vec>, - pub nodes: Vec, -} - -impl PackageTable { - pub fn new(container: &str, packages: &[FlagPackage]) -> Result { - // create table - let num_packages = packages.len() as u32; - let num_buckets = storage::get_table_size(num_packages)?; - let mut table = Self { - header: PackageTableHeader::new(container, num_packages), - buckets: vec![None; num_buckets as usize], - nodes: packages.iter().map(|pkg| PackageTableNode::new(pkg, num_buckets)).collect(), - }; - - // initialize all header fields - table.header.bucket_offset = table.header.as_bytes().len() as u32; - table.header.node_offset = table.header.bucket_offset + num_buckets * 4; - table.header.file_size = table.header.node_offset - + table.nodes.iter().map(|x| x.as_bytes().len()).sum::() as u32; - - // sort nodes by bucket index for efficiency - table.nodes.sort_by(|a, b| a.bucket_index.cmp(&b.bucket_index)); - - // fill all node offset - let mut offset = table.header.node_offset; - for i in 0..table.nodes.len() { - let node_bucket_idx = table.nodes[i].bucket_index; - let next_node_bucket_idx = if i + 1 < table.nodes.len() { - Some(table.nodes[i + 1].bucket_index) - } else { - None - }; - - if table.buckets[node_bucket_idx as usize].is_none() { - table.buckets[node_bucket_idx as usize] = Some(offset); - } - offset += table.nodes[i].as_bytes().len() as u32; - - if let Some(index) = next_node_bucket_idx { - if index == node_bucket_idx { - table.nodes[i].next_offset = Some(offset); - } + if let Some(index) = next_node_bucket_idx { + if index == node_bucket_idx { + table.nodes[i].next_offset = Some(offset); } } - - Ok(table) } - pub fn as_bytes(&self) -> Vec { - [ - self.header.as_bytes(), - self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::>().concat(), - self.nodes.iter().map(|v| v.as_bytes()).collect::>().concat(), - ] - .concat() - } + Ok(table) } #[cfg(test)] mod tests { use super::*; - use crate::storage::{ - group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes, - tests::read_u32_from_bytes, - }; - - impl PackageTableHeader { - // test only method to deserialize back into the header struct - fn from_bytes(bytes: &[u8]) -> Result { - let mut head = 0; - Ok(Self { - version: read_u32_from_bytes(bytes, &mut head)?, - container: read_str_from_bytes(bytes, &mut head)?, - file_size: read_u32_from_bytes(bytes, &mut head)?, - num_packages: read_u32_from_bytes(bytes, &mut head)?, - bucket_offset: read_u32_from_bytes(bytes, &mut head)?, - node_offset: read_u32_from_bytes(bytes, &mut head)?, - }) - } - } - - impl PackageTableNode { - // test only method to deserialize back into the node struct - fn from_bytes(bytes: &[u8], num_buckets: u32) -> Result { - let mut head = 0; - let mut node = Self { - package_name: read_str_from_bytes(bytes, &mut head)?, - package_id: read_u32_from_bytes(bytes, &mut head)?, - boolean_offset: read_u32_from_bytes(bytes, &mut head)?, - next_offset: match read_u32_from_bytes(bytes, &mut head)? { - 0 => None, - val => Some(val), - }, - bucket_index: 0, - }; - node.bucket_index = storage::get_bucket_index(&node.package_name, num_buckets); - Ok(node) - } - } - - impl PackageTable { - // test only method to deserialize back into the table struct - fn from_bytes(bytes: &[u8]) -> Result { - let header = PackageTableHeader::from_bytes(bytes)?; - let num_packages = header.num_packages; - let num_buckets = storage::get_table_size(num_packages)?; - let mut head = header.as_bytes().len(); - let buckets = (0..num_buckets) - .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { - 0 => None, - val => Some(val), - }) - .collect(); - let nodes = (0..num_packages) - .map(|_| { - let node = PackageTableNode::from_bytes(&bytes[head..], num_buckets).unwrap(); - head += node.as_bytes().len(); - node - }) - .collect(); - - let table = Self { header, buckets, nodes }; - Ok(table) - } - } + use crate::storage::{group_flags_by_package, tests::parse_all_test_flags}; pub fn create_test_package_table() -> Result { let caches = parse_all_test_flags(); let packages = group_flags_by_package(caches.iter()); - PackageTable::new("system", &packages) + create_package_table("system", &packages) } #[test] @@ -233,7 +105,7 @@ mod tests { let header: &PackageTableHeader = &package_table.as_ref().unwrap().header; let expected_header = PackageTableHeader { - version: storage::FILE_VERSION, + version: FILE_VERSION, container: String::from("system"), file_size: 208, num_packages: 3, @@ -273,27 +145,4 @@ mod tests { }; assert_eq!(nodes[2], third_node_expected); } - - #[test] - // this test point locks down the table serialization - fn test_serialization() { - let package_table = create_test_package_table().unwrap(); - - let header: &PackageTableHeader = &package_table.header; - let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes()); - assert!(reinterpreted_header.is_ok()); - assert_eq!(header, &reinterpreted_header.unwrap()); - - let nodes: &Vec = &package_table.nodes; - let num_buckets = storage::get_table_size(header.num_packages).unwrap(); - for node in nodes.iter() { - let reinterpreted_node = PackageTableNode::from_bytes(&node.as_bytes(), num_buckets); - assert!(reinterpreted_node.is_ok()); - assert_eq!(node, &reinterpreted_node.unwrap()); - } - - let reinterpreted_table = PackageTable::from_bytes(&package_table.as_bytes()); - assert!(reinterpreted_table.is_ok()); - assert_eq!(&package_table, &reinterpreted_table.unwrap()); - } } diff --git a/tools/aconfig/aconfig_storage_file/Android.bp b/tools/aconfig/aconfig_storage_file/Android.bp new file mode 100644 index 0000000000..98d7bebd94 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/Android.bp @@ -0,0 +1,14 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +rust_library { + name: "libaconfig_storage_file", + srcs: ["src/lib.rs"], + crate_name: "aconfig_storage_file", + host_supported: true, + lints: "none", + rustlibs: [ + "libanyhow", + ], +} diff --git a/tools/aconfig/aconfig_storage_file/Cargo.toml b/tools/aconfig/aconfig_storage_file/Cargo.toml new file mode 100644 index 0000000000..03c7309c0d --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "aconfig_storage_file" +version = "0.1.0" +edition = "2021" + +[features] +default = ["cargo"] +cargo = [] + +[dependencies] +anyhow = "1.0.69" diff --git a/tools/aconfig/aconfig_storage_file/src/flag_table.rs b/tools/aconfig/aconfig_storage_file/src/flag_table.rs new file mode 100644 index 0000000000..27a97d1255 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/flag_table.rs @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::{read_str_from_bytes, read_u16_from_bytes, read_u32_from_bytes}; +use anyhow::Result; + +#[derive(PartialEq, Debug)] +pub struct FlagTableHeader { + pub version: u32, + pub container: String, + pub file_size: u32, + pub num_flags: u32, + pub bucket_offset: u32, + pub node_offset: u32, +} + +impl FlagTableHeader { + pub fn as_bytes(&self) -> Vec { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_flags.to_le_bytes()); + result.extend_from_slice(&self.bucket_offset.to_le_bytes()); + result.extend_from_slice(&self.node_offset.to_le_bytes()); + result + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut head = 0; + Ok(Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_flags: read_u32_from_bytes(bytes, &mut head)?, + bucket_offset: read_u32_from_bytes(bytes, &mut head)?, + node_offset: read_u32_from_bytes(bytes, &mut head)?, + }) + } +} + +#[derive(PartialEq, Debug, Clone)] +pub struct FlagTableNode { + pub package_id: u32, + pub flag_name: String, + pub flag_type: u16, + pub flag_id: u16, + pub next_offset: Option, + pub bucket_index: u32, +} + +impl FlagTableNode { + pub fn as_bytes(&self) -> Vec { + let mut result = Vec::new(); + result.extend_from_slice(&self.package_id.to_le_bytes()); + let name_bytes = self.flag_name.as_bytes(); + result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(name_bytes); + result.extend_from_slice(&self.flag_type.to_le_bytes()); + result.extend_from_slice(&self.flag_id.to_le_bytes()); + result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); + result + } + + pub fn from_bytes(bytes: &[u8], num_buckets: u32) -> Result { + let mut head = 0; + let mut node = Self { + package_id: read_u32_from_bytes(bytes, &mut head)?, + flag_name: read_str_from_bytes(bytes, &mut head)?, + flag_type: read_u16_from_bytes(bytes, &mut head)?, + flag_id: read_u16_from_bytes(bytes, &mut head)?, + next_offset: match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), + }, + bucket_index: 0, + }; + let full_flag_name = node.package_id.to_string() + "/" + &node.flag_name; + node.bucket_index = crate::get_bucket_index(&full_flag_name, num_buckets); + Ok(node) + } +} + +#[derive(PartialEq, Debug)] +pub struct FlagTable { + pub header: FlagTableHeader, + pub buckets: Vec>, + pub nodes: Vec, +} + +impl FlagTable { + pub fn as_bytes(&self) -> Vec { + [ + self.header.as_bytes(), + self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::>().concat(), + self.nodes.iter().map(|v| v.as_bytes()).collect::>().concat(), + ] + .concat() + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let header = FlagTableHeader::from_bytes(bytes)?; + let num_flags = header.num_flags; + let num_buckets = crate::get_table_size(num_flags)?; + let mut head = header.as_bytes().len(); + let buckets = (0..num_buckets) + .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { + 0 => None, + val => Some(val), + }) + .collect(); + let nodes = (0..num_flags) + .map(|_| { + let node = FlagTableNode::from_bytes(&bytes[head..], num_buckets).unwrap(); + head += node.as_bytes().len(); + node + }) + .collect(); + + let table = Self { header, buckets, nodes }; + Ok(table) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + impl FlagTableNode { + // create test baseline, syntactic sugar + fn new_expected( + package_id: u32, + flag_name: &str, + flag_type: u16, + flag_id: u16, + next_offset: Option, + bucket_index: u32, + ) -> Self { + Self { + package_id, + flag_name: flag_name.to_string(), + flag_type, + flag_id, + next_offset, + bucket_index, + } + } + } + + pub fn create_test_flag_table() -> Result { + let header = FlagTableHeader { + version: crate::FILE_VERSION, + container: String::from("system"), + file_size: 320, + num_flags: 8, + bucket_offset: 30, + node_offset: 98, + }; + let buckets: Vec> = vec![ + Some(98), + Some(124), + None, + None, + None, + Some(177), + None, + Some(203), + None, + Some(261), + None, + None, + None, + None, + None, + Some(293), + None, + ]; + let nodes = vec![ + FlagTableNode::new_expected(0, "enabled_ro", 1, 1, None, 0), + FlagTableNode::new_expected(0, "enabled_rw", 1, 2, Some(150), 1), + FlagTableNode::new_expected(1, "disabled_ro", 1, 0, None, 1), + FlagTableNode::new_expected(2, "enabled_ro", 1, 1, None, 5), + FlagTableNode::new_expected(1, "enabled_fixed_ro", 1, 1, Some(235), 7), + FlagTableNode::new_expected(1, "enabled_ro", 1, 2, None, 7), + FlagTableNode::new_expected(2, "enabled_fixed_ro", 1, 0, None, 9), + FlagTableNode::new_expected(0, "disabled_rw", 1, 0, None, 15), + ]; + Ok(FlagTable { header, buckets, nodes }) + } + + #[test] + // this test point locks down the table serialization + fn test_serialization() { + let flag_table = create_test_flag_table().unwrap(); + + let header: &FlagTableHeader = &flag_table.header; + let reinterpreted_header = FlagTableHeader::from_bytes(&header.as_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec = &flag_table.nodes; + let num_buckets = crate::get_table_size(header.num_flags).unwrap(); + for node in nodes.iter() { + let reinterpreted_node = FlagTableNode::from_bytes(&node.as_bytes(), num_buckets); + assert!(reinterpreted_node.is_ok()); + assert_eq!(node, &reinterpreted_node.unwrap()); + } + + let reinterpreted_table = FlagTable::from_bytes(&flag_table.as_bytes()); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&flag_table, &reinterpreted_table.unwrap()); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/flag_value.rs b/tools/aconfig/aconfig_storage_file/src/flag_value.rs new file mode 100644 index 0000000000..9679c55bf2 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/flag_value.rs @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; +use anyhow::Result; + +#[derive(PartialEq, Debug)] +pub struct FlagValueHeader { + pub version: u32, + pub container: String, + pub file_size: u32, + pub num_flags: u32, + pub boolean_value_offset: u32, +} + +impl FlagValueHeader { + pub fn as_bytes(&self) -> Vec { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_flags.to_le_bytes()); + result.extend_from_slice(&self.boolean_value_offset.to_le_bytes()); + result + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut head = 0; + Ok(Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_flags: read_u32_from_bytes(bytes, &mut head)?, + boolean_value_offset: read_u32_from_bytes(bytes, &mut head)?, + }) + } +} + +#[derive(PartialEq, Debug)] +pub struct FlagValueList { + pub header: FlagValueHeader, + pub booleans: Vec, +} + +impl FlagValueList { + pub fn as_bytes(&self) -> Vec { + [ + self.header.as_bytes(), + self.booleans.iter().map(|&v| u8::from(v).to_le_bytes()).collect::>().concat(), + ] + .concat() + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let header = FlagValueHeader::from_bytes(bytes)?; + let num_flags = header.num_flags; + let mut head = header.as_bytes().len(); + let booleans = + (0..num_flags).map(|_| read_u8_from_bytes(bytes, &mut head).unwrap() == 1).collect(); + let list = Self { header, booleans }; + Ok(list) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub fn create_test_flag_value_list() -> Result { + let header = FlagValueHeader { + version: crate::FILE_VERSION, + container: String::from("system"), + file_size: 34, + num_flags: 8, + boolean_value_offset: 26, + }; + let booleans: Vec = vec![false, true, false, false, true, true, false, true]; + Ok(FlagValueList { header, booleans }) + } + + #[test] + // this test point locks down the value list serialization + fn test_serialization() { + let flag_value_list = create_test_flag_value_list().unwrap(); + + let header: &FlagValueHeader = &flag_value_list.header; + let reinterpreted_header = FlagValueHeader::from_bytes(&header.as_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let reinterpreted_value_list = FlagValueList::from_bytes(&flag_value_list.as_bytes()); + assert!(reinterpreted_value_list.is_ok()); + assert_eq!(&flag_value_list, &reinterpreted_value_list.unwrap()); + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/lib.rs b/tools/aconfig/aconfig_storage_file/src/lib.rs new file mode 100644 index 0000000000..078b5aaba3 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/lib.rs @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pub mod flag_table; +pub mod flag_value; +pub mod package_table; + +use anyhow::{anyhow, Result}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode}; +pub use crate::flag_value::{FlagValueHeader, FlagValueList}; +pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode}; + +pub const FILE_VERSION: u32 = 1; + +pub const HASH_PRIMES: [u32; 29] = [ + 7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241, + 786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611, + 402653189, 805306457, 1610612741, +]; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageFileSelection { + PackageMap, + FlagMap, + FlagVal, +} + +impl TryFrom<&str> for StorageFileSelection { + type Error = anyhow::Error; + + fn try_from(value: &str) -> std::result::Result { + match value { + "package_map" => Ok(Self::PackageMap), + "flag_map" => Ok(Self::FlagMap), + "flag_val" => Ok(Self::FlagVal), + _ => Err(anyhow!("Invalid storage file to create")), + } + } +} + +/// Get the right hash table size given number of entries in the table. Use a +/// load factor of 0.5 for performance. +pub fn get_table_size(entries: u32) -> Result { + HASH_PRIMES + .iter() + .find(|&&num| num >= 2 * entries) + .copied() + .ok_or(anyhow!("Number of packages is too large")) +} + +/// Get the corresponding bucket index given the key and number of buckets +pub fn get_bucket_index(val: &T, num_buckets: u32) -> u32 { + let mut s = DefaultHasher::new(); + val.hash(&mut s); + (s.finish() % num_buckets as u64) as u32 +} + +/// Read and parse bytes as u8 +pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result { + let val = u8::from_le_bytes(buf[*head..*head + 1].try_into()?); + *head += 1; + Ok(val) +} + +/// Read and parse bytes as u16 +pub fn read_u16_from_bytes(buf: &[u8], head: &mut usize) -> Result { + let val = u16::from_le_bytes(buf[*head..*head + 2].try_into()?); + *head += 2; + Ok(val) +} + +/// Read and parse bytes as u32 +pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result { + let val = u32::from_le_bytes(buf[*head..*head + 4].try_into()?); + *head += 4; + Ok(val) +} + +/// Read and parse bytes as string +pub fn read_str_from_bytes(buf: &[u8], head: &mut usize) -> Result { + let num_bytes = read_u32_from_bytes(buf, head)? as usize; + let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())?; + *head += num_bytes; + Ok(val) +} diff --git a/tools/aconfig/aconfig_storage_file/src/package_table.rs b/tools/aconfig/aconfig_storage_file/src/package_table.rs new file mode 100644 index 0000000000..b454430b04 --- /dev/null +++ b/tools/aconfig/aconfig_storage_file/src/package_table.rs @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use crate::{read_str_from_bytes, read_u32_from_bytes}; +use anyhow::Result; + +#[derive(PartialEq, Debug)] +pub struct PackageTableHeader { + pub version: u32, + pub container: String, + pub file_size: u32, + pub num_packages: u32, + pub bucket_offset: u32, + pub node_offset: u32, +} + +impl PackageTableHeader { + pub fn as_bytes(&self) -> Vec { + let mut result = Vec::new(); + result.extend_from_slice(&self.version.to_le_bytes()); + let container_bytes = self.container.as_bytes(); + result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(container_bytes); + result.extend_from_slice(&self.file_size.to_le_bytes()); + result.extend_from_slice(&self.num_packages.to_le_bytes()); + result.extend_from_slice(&self.bucket_offset.to_le_bytes()); + result.extend_from_slice(&self.node_offset.to_le_bytes()); + result + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut head = 0; + Ok(Self { + version: read_u32_from_bytes(bytes, &mut head)?, + container: read_str_from_bytes(bytes, &mut head)?, + file_size: read_u32_from_bytes(bytes, &mut head)?, + num_packages: read_u32_from_bytes(bytes, &mut head)?, + bucket_offset: read_u32_from_bytes(bytes, &mut head)?, + node_offset: read_u32_from_bytes(bytes, &mut head)?, + }) + } +} + +#[derive(PartialEq, Debug)] +pub struct PackageTableNode { + pub package_name: String, + pub package_id: u32, + // offset of the first boolean flag in this flag package with respect to the start of + // boolean flag value array in the flag value file + pub boolean_offset: u32, + pub next_offset: Option, + pub bucket_index: u32, +} + +impl PackageTableNode { + pub fn as_bytes(&self) -> Vec { + let mut result = Vec::new(); + let name_bytes = self.package_name.as_bytes(); + result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); + result.extend_from_slice(name_bytes); + result.extend_from_slice(&self.package_id.to_le_bytes()); + result.extend_from_slice(&self.boolean_offset.to_le_bytes()); + result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes()); + result + } + + pub fn from_bytes(bytes: &[u8], num_buckets: u32) -> Result { + let mut head = 0; + let mut node = Self { + package_name: read_str_from_bytes(bytes, &mut head)?, + package_id: read_u32_from_bytes(bytes, &mut head)?, + boolean_offset: read_u32_from_bytes(bytes, &mut head)?, + next_offset: match read_u32_from_bytes(bytes, &mut head)? { + 0 => None, + val => Some(val), + }, + bucket_index: 0, + }; + node.bucket_index = crate::get_bucket_index(&node.package_name, num_buckets); + Ok(node) + } +} + +#[derive(PartialEq, Debug)] +pub struct PackageTable { + pub header: PackageTableHeader, + pub buckets: Vec>, + pub nodes: Vec, +} + +impl PackageTable { + pub fn as_bytes(&self) -> Vec { + [ + self.header.as_bytes(), + self.buckets.iter().map(|v| v.unwrap_or(0).to_le_bytes()).collect::>().concat(), + self.nodes.iter().map(|v| v.as_bytes()).collect::>().concat(), + ] + .concat() + } + + pub fn from_bytes(bytes: &[u8]) -> Result { + let header = PackageTableHeader::from_bytes(bytes)?; + let num_packages = header.num_packages; + let num_buckets = crate::get_table_size(num_packages)?; + let mut head = header.as_bytes().len(); + let buckets = (0..num_buckets) + .map(|_| match read_u32_from_bytes(bytes, &mut head).unwrap() { + 0 => None, + val => Some(val), + }) + .collect(); + let nodes = (0..num_packages) + .map(|_| { + let node = PackageTableNode::from_bytes(&bytes[head..], num_buckets).unwrap(); + head += node.as_bytes().len(); + node + }) + .collect(); + + let table = Self { header, buckets, nodes }; + Ok(table) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub fn create_test_package_table() -> Result { + let header = PackageTableHeader { + version: crate::FILE_VERSION, + container: String::from("system"), + file_size: 208, + num_packages: 3, + bucket_offset: 30, + node_offset: 58, + }; + let buckets: Vec> = vec![Some(58), None, None, Some(108), None, None, None]; + let first_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_2"), + package_id: 1, + boolean_offset: 3, + next_offset: None, + bucket_index: 0, + }; + let second_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_1"), + package_id: 0, + boolean_offset: 0, + next_offset: Some(158), + bucket_index: 3, + }; + let third_node = PackageTableNode { + package_name: String::from("com.android.aconfig.storage.test_4"), + package_id: 2, + boolean_offset: 6, + next_offset: None, + bucket_index: 3, + }; + let nodes = vec![first_node, second_node, third_node]; + Ok(PackageTable { header, buckets, nodes }) + } + + #[test] + // this test point locks down the table serialization + fn test_serialization() { + let package_table = create_test_package_table().unwrap(); + + let header: &PackageTableHeader = &package_table.header; + let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes()); + assert!(reinterpreted_header.is_ok()); + assert_eq!(header, &reinterpreted_header.unwrap()); + + let nodes: &Vec = &package_table.nodes; + let num_buckets = crate::get_table_size(header.num_packages).unwrap(); + for node in nodes.iter() { + let reinterpreted_node = PackageTableNode::from_bytes(&node.as_bytes(), num_buckets); + assert!(reinterpreted_node.is_ok()); + assert_eq!(node, &reinterpreted_node.unwrap()); + } + + let reinterpreted_table = PackageTable::from_bytes(&package_table.as_bytes()); + assert!(reinterpreted_table.is_ok()); + assert_eq!(&package_table, &reinterpreted_table.unwrap()); + } +}