diff --git a/tools/aconfig/aconfig_storage_file/Android.bp b/tools/aconfig/aconfig_storage_file/Android.bp index 53b693f17b..8c1e2cb9db 100644 --- a/tools/aconfig/aconfig_storage_file/Android.bp +++ b/tools/aconfig/aconfig_storage_file/Android.bp @@ -13,6 +13,9 @@ rust_defaults { "libonce_cell", "libprotobuf", "libtempfile", + "libmemmap2", + "libcxx", + "libthiserror", ], } @@ -23,10 +26,60 @@ rust_library { defaults: ["aconfig_storage_file.defaults"], } +genrule { + name: "ro.package.map", + out: ["tests/tmp.ro.package.map"], + srcs: ["tests/package.map"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)", +} + +genrule { + name: "ro.flag.map", + out: ["tests/tmp.ro.flag.map"], + srcs: ["tests/flag.map"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)", +} + +genrule { + name: "ro.flag.val", + out: ["tests/tmp.ro.flag.val"], + srcs: ["tests/flag.val"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)", +} + +genrule { + name: "rw.package.map", + out: ["tests/tmp.rw.package.map"], + srcs: ["tests/package.map"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)", +} + +genrule { + name: "rw.flag.map", + out: ["tests/tmp.rw.flag.map"], + srcs: ["tests/flag.map"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)", +} + +genrule { + name: "rw.flag.val", + out: ["tests/tmp.rw.flag.val"], + srcs: ["tests/flag.val"], + cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)", +} + rust_test_host { name: "aconfig_storage_file.test", test_suites: ["general-tests"], defaults: ["aconfig_storage_file.defaults"], + data: [ + ":ro.package.map", + ":ro.flag.map", + ":ro.flag.val", + ":rw.package.map", + ":rw.flag.map", + ":rw.flag.val", + ], } rust_protobuf { diff --git a/tools/aconfig/aconfig_storage_file/Cargo.toml b/tools/aconfig/aconfig_storage_file/Cargo.toml index 54ba6c7840..296d068b0f 100644 --- a/tools/aconfig/aconfig_storage_file/Cargo.toml +++ b/tools/aconfig/aconfig_storage_file/Cargo.toml @@ -13,6 +13,8 @@ memmap2 = "0.8.0" protobuf = "3.2.0" once_cell = "1.19.0" tempfile = "3.9.0" +cxx = "1.0" +thiserror = "1.0.56" [build-dependencies] protobuf-codegen = "3.2.0" diff --git a/tools/aconfig/aconfig_storage_file/src/flag_table.rs b/tools/aconfig/aconfig_storage_file/src/flag_table.rs index dfbd9de830..108804ec26 100644 --- a/tools/aconfig/aconfig_storage_file/src/flag_table.rs +++ b/tools/aconfig/aconfig_storage_file/src/flag_table.rs @@ -17,8 +17,10 @@ //! flag table module defines the flag table file format and methods for serialization //! and deserialization +use crate::AconfigStorageError::{self, BytesParseFail, HigherStorageFileVersion}; use crate::{get_bucket_index, read_str_from_bytes, read_u16_from_bytes, read_u32_from_bytes}; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; +pub type FlagOffset = u16; /// Flag table header struct #[derive(PartialEq, Debug)] @@ -47,7 +49,7 @@ impl FlagTableHeader { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let mut head = 0; Ok(Self { version: read_u32_from_bytes(bytes, &mut head)?, @@ -85,7 +87,7 @@ impl FlagTableNode { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let mut head = 0; let node = Self { package_id: read_u32_from_bytes(bytes, &mut head)?, @@ -127,7 +129,7 @@ impl FlagTable { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let header = FlagTableHeader::from_bytes(bytes)?; let num_flags = header.num_flags; let num_buckets = crate::get_table_size(num_flags)?; @@ -144,7 +146,8 @@ impl FlagTable { head += node.as_bytes().len(); Ok(node) }) - .collect::>>()?; + .collect::, AconfigStorageError>>() + .map_err(|errmsg| BytesParseFail(anyhow!("fail to parse flag table: {}", errmsg)))?; let table = Self { header, buckets, nodes }; Ok(table) @@ -152,14 +155,18 @@ impl FlagTable { } /// Query flag within package offset -pub fn find_flag_offset(buf: &[u8], package_id: u32, flag: &str) -> Result> { +pub fn find_flag_offset( + buf: &[u8], + package_id: u32, + flag: &str, +) -> Result, AconfigStorageError> { let interpreted_header = FlagTableHeader::from_bytes(buf)?; if interpreted_header.version > crate::FILE_VERSION { - return Err(anyhow!( + return Err(HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, crate::FILE_VERSION - )); + ))); } let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4; @@ -202,7 +209,7 @@ mod tests { } } - pub fn create_test_flag_table() -> Result { + pub fn create_test_flag_table() -> FlagTable { let header = FlagTableHeader { version: crate::FILE_VERSION, container: String::from("system"), @@ -240,13 +247,13 @@ mod tests { FlagTableNode::new_expected(2, "enabled_fixed_ro", 1, 0, None), FlagTableNode::new_expected(0, "disabled_rw", 1, 0, None), ]; - Ok(FlagTable { header, buckets, nodes }) + FlagTable { header, buckets, nodes } } #[test] // this test point locks down the table serialization fn test_serialization() { - let flag_table = create_test_flag_table().unwrap(); + let flag_table = create_test_flag_table(); let header: &FlagTableHeader = &flag_table.header; let reinterpreted_header = FlagTableHeader::from_bytes(&header.as_bytes()); @@ -267,7 +274,7 @@ mod tests { #[test] // this test point locks down table query fn test_flag_query() { - let flag_table = create_test_flag_table().unwrap().as_bytes(); + let flag_table = create_test_flag_table().as_bytes(); let baseline = vec![ (0, "enabled_ro", 1u16), (0, "enabled_rw", 2u16), @@ -288,7 +295,7 @@ mod tests { #[test] // this test point locks down table query of a non exist flag fn test_not_existed_flag_query() { - let flag_table = create_test_flag_table().unwrap().as_bytes(); + let flag_table = create_test_flag_table().as_bytes(); let flag_offset = find_flag_offset(&flag_table[..], 1, "disabled_fixed_ro").unwrap(); assert_eq!(flag_offset, None); let flag_offset = find_flag_offset(&flag_table[..], 2, "disabled_rw").unwrap(); @@ -298,14 +305,14 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut table = create_test_flag_table().unwrap(); + let mut table = create_test_flag_table(); table.header.version = crate::FILE_VERSION + 1; let flag_table = table.as_bytes(); let error = find_flag_offset(&flag_table[..], 0, "enabled_ro").unwrap_err(); assert_eq!( format!("{:?}", error), format!( - "Cannot read storage file with a higher version of {} with lib version {}", + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", crate::FILE_VERSION + 1, crate::FILE_VERSION ) diff --git a/tools/aconfig/aconfig_storage_file/src/flag_value.rs b/tools/aconfig/aconfig_storage_file/src/flag_value.rs index bb8892d5ca..0a6a37f979 100644 --- a/tools/aconfig/aconfig_storage_file/src/flag_value.rs +++ b/tools/aconfig/aconfig_storage_file/src/flag_value.rs @@ -17,8 +17,9 @@ //! flag value module defines the flag value file format and methods for serialization //! and deserialization +use crate::AconfigStorageError::{self, HigherStorageFileVersion, InvalidStorageFileOffset}; use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes}; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; /// Flag value header struct #[derive(PartialEq, Debug)] @@ -45,7 +46,7 @@ impl FlagValueHeader { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let mut head = 0; Ok(Self { version: read_u32_from_bytes(bytes, &mut head)?, @@ -75,7 +76,7 @@ impl FlagValueList { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let header = FlagValueHeader::from_bytes(bytes)?; let num_flags = header.num_flags; let mut head = header.as_bytes().len(); @@ -87,14 +88,14 @@ impl FlagValueList { } /// Query flag value -pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result { +pub fn find_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result { let interpreted_header = FlagValueHeader::from_bytes(buf)?; if interpreted_header.version > crate::FILE_VERSION { - return Err(anyhow!( + return Err(HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, crate::FILE_VERSION - )); + ))); } let mut head = (interpreted_header.boolean_value_offset + flag_offset) as usize; @@ -102,7 +103,9 @@ pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result { // TODO: right now, there is only boolean flags, with more flag value types added // later, the end of boolean flag value section should be updated (b/322826265). if head >= interpreted_header.file_size as usize { - return Err(anyhow!("Flag value offset goes beyond the end of the file.")); + return Err(InvalidStorageFileOffset(anyhow!( + "Flag value offset goes beyond the end of the file." + ))); } let val = read_u8_from_bytes(buf, &mut head)?; @@ -113,7 +116,7 @@ pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result { mod tests { use super::*; - pub fn create_test_flag_value_list() -> Result { + pub fn create_test_flag_value_list() -> FlagValueList { let header = FlagValueHeader { version: crate::FILE_VERSION, container: String::from("system"), @@ -122,13 +125,13 @@ mod tests { boolean_value_offset: 26, }; let booleans: Vec = vec![false, true, false, false, true, true, false, true]; - Ok(FlagValueList { header, booleans }) + FlagValueList { header, booleans } } #[test] // this test point locks down the value list serialization fn test_serialization() { - let flag_value_list = create_test_flag_value_list().unwrap(); + let flag_value_list = create_test_flag_value_list(); let header: &FlagValueHeader = &flag_value_list.header; let reinterpreted_header = FlagValueHeader::from_bytes(&header.as_bytes()); @@ -143,10 +146,10 @@ mod tests { #[test] // this test point locks down flag value query fn test_flag_value_query() { - let flag_value_list = create_test_flag_value_list().unwrap().as_bytes(); + let flag_value_list = create_test_flag_value_list().as_bytes(); let baseline: Vec = vec![false, true, false, false, true, true, false, true]; for (offset, expected_value) in baseline.into_iter().enumerate() { - let flag_value = get_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap(); + let flag_value = find_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap(); assert_eq!(flag_value, expected_value); } } @@ -154,22 +157,25 @@ mod tests { #[test] // this test point locks down query beyond the end of boolean section fn test_boolean_out_of_range() { - let flag_value_list = create_test_flag_value_list().unwrap().as_bytes(); - let error = get_boolean_flag_value(&flag_value_list[..], 8).unwrap_err(); - assert_eq!(format!("{:?}", error), "Flag value offset goes beyond the end of the file."); + let flag_value_list = create_test_flag_value_list().as_bytes(); + let error = find_boolean_flag_value(&flag_value_list[..], 8).unwrap_err(); + assert_eq!( + format!("{:?}", error), + "InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)" + ); } #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut value_list = create_test_flag_value_list().unwrap(); + let mut value_list = create_test_flag_value_list(); value_list.header.version = crate::FILE_VERSION + 1; let flag_value = value_list.as_bytes(); - let error = get_boolean_flag_value(&flag_value[..], 4).unwrap_err(); + let error = find_boolean_flag_value(&flag_value[..], 4).unwrap_err(); assert_eq!( format!("{:?}", error), format!( - "Cannot read storage file with a higher version of {} with lib version {}", + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", crate::FILE_VERSION + 1, crate::FILE_VERSION ) diff --git a/tools/aconfig/aconfig_storage_file/src/lib.rs b/tools/aconfig/aconfig_storage_file/src/lib.rs index a9f5e21e5c..e87207aa4c 100644 --- a/tools/aconfig/aconfig_storage_file/src/lib.rs +++ b/tools/aconfig/aconfig_storage_file/src/lib.rs @@ -15,37 +15,56 @@ */ //! `aconfig_storage_file` is a crate that defines aconfig storage file format, it -//! also includes apis to read flags from storage files +//! also includes apis to read flags from storage files. It provides three apis to +//! interface with storage files: +//! +//! 1, function to get package flag value start offset +//! pub fn get_package_offset(container: &str, package: &str) -> `Result>>` +//! +//! 2, function to get flag offset within a specific package +//! pub fn get_flag_offset(container: &str, package_id: u32, flag: &str) -> `Result>>` +//! +//! 3, function to get the actual flag value given the global offset (combined package and +//! flag offset). +//! pub fn get_boolean_flag_value(container: &str, offset: u32) -> `Result` +//! +//! Note these are low level apis that are expected to be only used in auto generated flag +//! apis. DO NOT DIRECTLY USE THESE APIS IN YOUR SOURCE CODE. For auto generated flag apis +//! please refer to the g3doc go/android-flags pub mod flag_table; pub mod flag_value; -pub mod package_table; - -#[cfg(feature = "cargo")] pub mod mapped_file; +pub mod package_table; +pub mod protos; -mod protos; #[cfg(test)] mod test_utils; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode}; +pub use crate::flag_table::{FlagOffset, FlagTable, FlagTableHeader, FlagTableNode}; pub use crate::flag_value::{FlagValueHeader, FlagValueList}; -pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode}; +pub use crate::package_table::{PackageOffset, PackageTable, PackageTableHeader, PackageTableNode}; +pub use crate::protos::ProtoStorageFiles; + +use crate::AconfigStorageError::{BytesParseFail, HashTableSizeLimit}; /// Storage file version pub const FILE_VERSION: u32 = 1; /// Good hash table prime number -pub const HASH_PRIMES: [u32; 29] = [ +pub(crate) const HASH_PRIMES: [u32; 29] = [ 7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241, 786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611, 402653189, 805306457, 1610612741, ]; +/// Storage file location pb file +pub const STORAGE_LOCATION_FILE: &str = "/metadata/aconfig/storage_files.pb"; + /// Storage file type enum #[derive(Clone, Debug, PartialEq, Eq)] pub enum StorageFileSelection { @@ -69,46 +88,300 @@ impl TryFrom<&str> for StorageFileSelection { /// Get the right hash table size given number of entries in the table. Use a /// load factor of 0.5 for performance. -pub fn get_table_size(entries: u32) -> Result { +pub fn get_table_size(entries: u32) -> Result { HASH_PRIMES .iter() .find(|&&num| num >= 2 * entries) .copied() - .ok_or(anyhow!("Number of packages is too large")) + .ok_or(HashTableSizeLimit(anyhow!("Number of items in a hash table exceeds limit"))) } /// Get the corresponding bucket index given the key and number of buckets -pub fn get_bucket_index(val: &T, num_buckets: u32) -> u32 { +pub(crate) fn get_bucket_index(val: &T, num_buckets: u32) -> u32 { let mut s = DefaultHasher::new(); val.hash(&mut s); (s.finish() % num_buckets as u64) as u32 } /// Read and parse bytes as u8 -pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u8::from_le_bytes(buf[*head..*head + 1].try_into()?); +pub(crate) fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result { + let val = + u8::from_le_bytes(buf[*head..*head + 1].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u8 from bytes: {}", errmsg)) + })?); *head += 1; Ok(val) } /// Read and parse bytes as u16 -pub fn read_u16_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u16::from_le_bytes(buf[*head..*head + 2].try_into()?); +pub(crate) fn read_u16_from_bytes( + buf: &[u8], + head: &mut usize, +) -> Result { + let val = + u16::from_le_bytes(buf[*head..*head + 2].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u16 from bytes: {}", errmsg)) + })?); *head += 2; Ok(val) } /// Read and parse bytes as u32 -pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result { - let val = u32::from_le_bytes(buf[*head..*head + 4].try_into()?); +pub(crate) fn read_u32_from_bytes( + buf: &[u8], + head: &mut usize, +) -> Result { + let val = + u32::from_le_bytes(buf[*head..*head + 4].try_into().map_err(|errmsg| { + BytesParseFail(anyhow!("fail to parse u32 from bytes: {}", errmsg)) + })?); *head += 4; Ok(val) } /// Read and parse bytes as string -pub fn read_str_from_bytes(buf: &[u8], head: &mut usize) -> Result { +pub(crate) fn read_str_from_bytes( + buf: &[u8], + head: &mut usize, +) -> Result { let num_bytes = read_u32_from_bytes(buf, head)? as usize; - let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())?; + let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec()) + .map_err(|errmsg| BytesParseFail(anyhow!("fail to parse string from bytes: {}", errmsg)))?; *head += num_bytes; Ok(val) } + +/// Storage query api error +#[non_exhaustive] +#[derive(thiserror::Error, Debug)] +pub enum AconfigStorageError { + #[error("failed to read the file")] + FileReadFail(#[source] anyhow::Error), + + #[error("fail to parse protobuf")] + ProtobufParseFail(#[source] anyhow::Error), + + #[error("storage files not found for this container")] + StorageFileNotFound(#[source] anyhow::Error), + + #[error("fail to map storage file")] + MapFileFail(#[source] anyhow::Error), + + #[error("number of items in hash table exceed limit")] + HashTableSizeLimit(#[source] anyhow::Error), + + #[error("failed to parse bytes into data")] + BytesParseFail(#[source] anyhow::Error), + + #[error("cannot parse storage files with a higher version")] + HigherStorageFileVersion(#[source] anyhow::Error), + + #[error("invalid storage file byte offset")] + InvalidStorageFileOffset(#[source] anyhow::Error), +} + +/// Get package start offset implementation +pub fn get_package_offset_impl( + pb_file: &str, + container: &str, + package: &str, +) -> Result, AconfigStorageError> { + let mapped_file = + crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::PackageMap)?; + crate::package_table::find_package_offset(&mapped_file, package) +} + +/// Get flag offset implementation +pub fn get_flag_offset_impl( + pb_file: &str, + container: &str, + package_id: u32, + flag: &str, +) -> Result, AconfigStorageError> { + let mapped_file = + crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::FlagMap)?; + crate::flag_table::find_flag_offset(&mapped_file, package_id, flag) +} + +/// Get boolean flag value implementation +pub fn get_boolean_flag_value_impl( + pb_file: &str, + container: &str, + offset: u32, +) -> Result { + let mapped_file = + crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::FlagVal)?; + crate::flag_value::find_boolean_flag_value(&mapped_file, offset) +} + +/// Get package start offset for flags given the container and package name. +/// +/// This function would map the corresponding package map file if has not been mapped yet, +/// and then look for the target package in this mapped file. +/// +/// If a package is found, it returns Ok(Some(PackageOffset)) +/// If a package is not found, it returns Ok(None) +/// If errors out such as no such package map file is found, it returns an Err(errmsg) +pub fn get_package_offset( + container: &str, + package: &str, +) -> Result, AconfigStorageError> { + get_package_offset_impl(STORAGE_LOCATION_FILE, container, package) +} + +/// Get flag offset within a package given the container name, package id and flag name. +/// +/// This function would map the corresponding flag map file if has not been mapped yet, +/// and then look for the target flag in this mapped file. +/// +/// If a flag is found, it returns Ok(Some(u16)) +/// If a flag is not found, it returns Ok(None) +/// If errors out such as no such flag map file is found, it returns an Err(errmsg) +pub fn get_flag_offset( + container: &str, + package_id: u32, + flag: &str, +) -> Result, AconfigStorageError> { + get_flag_offset_impl(STORAGE_LOCATION_FILE, container, package_id, flag) +} + +/// Get the boolean flag value given the container name and flag global offset +/// +/// This function would map the corresponding flag value file if has not been mapped yet, +/// and then look for the target flag value at the specified offset. +/// +/// If flag value file is successfully mapped and the provide offset is valid, it returns +/// the boolean flag value, otherwise it returns the error message. +pub fn get_boolean_flag_value(container: &str, offset: u32) -> Result { + get_boolean_flag_value_impl(STORAGE_LOCATION_FILE, container, offset) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + create_temp_storage_files_for_test, get_binary_storage_proto_bytes, + set_temp_storage_files_to_read_only, write_bytes_to_temp_file, + }; + + #[test] + // this test point locks down flag package offset query + fn test_package_offset_query() { + #[cfg(feature = "cargo")] + create_temp_storage_files_for_test(); + + set_temp_storage_files_to_read_only(); + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.ro.flag.map" + flag_val: "./tests/tmp.ro.flag.val" + timestamp: 12345 +} +"#; + let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap(); + let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap(); + let file_full_path = file.path().display().to_string(); + + let package_offset = get_package_offset_impl( + &file_full_path, + "system", + "com.android.aconfig.storage.test_1", + ) + .unwrap() + .unwrap(); + let expected_package_offset = PackageOffset { package_id: 0, boolean_offset: 0 }; + assert_eq!(package_offset, expected_package_offset); + + let package_offset = get_package_offset_impl( + &file_full_path, + "system", + "com.android.aconfig.storage.test_2", + ) + .unwrap() + .unwrap(); + let expected_package_offset = PackageOffset { package_id: 1, boolean_offset: 3 }; + assert_eq!(package_offset, expected_package_offset); + + let package_offset = get_package_offset_impl( + &file_full_path, + "system", + "com.android.aconfig.storage.test_4", + ) + .unwrap() + .unwrap(); + let expected_package_offset = PackageOffset { package_id: 2, boolean_offset: 6 }; + assert_eq!(package_offset, expected_package_offset); + } + + #[test] + // this test point locks down flag offset query + fn test_flag_offset_query() { + #[cfg(feature = "cargo")] + create_temp_storage_files_for_test(); + + set_temp_storage_files_to_read_only(); + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.ro.flag.map" + flag_val: "./tests/tmp.ro.flag.val" + timestamp: 12345 +} +"#; + let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap(); + let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap(); + let file_full_path = file.path().display().to_string(); + + let baseline = vec![ + (0, "enabled_ro", 1u16), + (0, "enabled_rw", 2u16), + (1, "disabled_ro", 0u16), + (2, "enabled_ro", 1u16), + (1, "enabled_fixed_ro", 1u16), + (1, "enabled_ro", 2u16), + (2, "enabled_fixed_ro", 0u16), + (0, "disabled_rw", 0u16), + ]; + for (package_id, flag_name, expected_offset) in baseline.into_iter() { + let flag_offset = + get_flag_offset_impl(&file_full_path, "system", package_id, flag_name) + .unwrap() + .unwrap(); + assert_eq!(flag_offset, expected_offset); + } + } + + #[test] + // this test point locks down flag offset query + fn test_flag_value_query() { + #[cfg(feature = "cargo")] + create_temp_storage_files_for_test(); + + set_temp_storage_files_to_read_only(); + let text_proto = r#" +files { + version: 0 + container: "system" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.ro.flag.map" + flag_val: "./tests/tmp.ro.flag.val" + timestamp: 12345 +} +"#; + let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap(); + let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap(); + let file_full_path = file.path().display().to_string(); + + let baseline: Vec = vec![false; 8]; + for (offset, expected_value) in baseline.into_iter().enumerate() { + let flag_value = + get_boolean_flag_value_impl(&file_full_path, "system", offset as u32).unwrap(); + assert_eq!(flag_value, expected_value); + } + } +} diff --git a/tools/aconfig/aconfig_storage_file/src/mapped_file.rs b/tools/aconfig/aconfig_storage_file/src/mapped_file.rs index 4f65df0197..ad0c66f36f 100644 --- a/tools/aconfig/aconfig_storage_file/src/mapped_file.rs +++ b/tools/aconfig/aconfig_storage_file/src/mapped_file.rs @@ -19,13 +19,16 @@ use std::fs::File; use std::io::{BufReader, Read}; use std::sync::{Arc, Mutex}; -use anyhow::{bail, ensure, Result}; +use anyhow::anyhow; use memmap2::Mmap; use once_cell::sync::Lazy; use crate::protos::{ storage_files::try_from_binary_proto, ProtoStorageFileInfo, ProtoStorageFiles, }; +use crate::AconfigStorageError::{ + self, FileReadFail, MapFileFail, ProtobufParseFail, StorageFileNotFound, +}; use crate::StorageFileSelection; /// Cache for already mapped files @@ -46,30 +49,43 @@ struct MappedStorageFileSet { fn find_container_storage_location( location_pb_file: &str, container: &str, -) -> Result { - let file = File::open(location_pb_file)?; +) -> Result { + let file = File::open(location_pb_file).map_err(|errmsg| { + FileReadFail(anyhow!("Failed to open file {}: {}", location_pb_file, errmsg)) + })?; let mut reader = BufReader::new(file); let mut bytes = Vec::new(); - reader.read_to_end(&mut bytes)?; - - let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes)?; + reader.read_to_end(&mut bytes).map_err(|errmsg| { + FileReadFail(anyhow!("Failed to read file {}: {}", location_pb_file, errmsg)) + })?; + let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes).map_err(|errmsg| { + ProtobufParseFail(anyhow!( + "Failed to parse storage location pb file {}: {}", + location_pb_file, + errmsg + )) + })?; for location_info in storage_locations.files.iter() { if location_info.container() == container { return Ok(location_info.clone()); } } - bail!("Storage file does not exist for {}", container) + Err(StorageFileNotFound(anyhow!("Storage file does not exist for {}", container))) } /// Verify the file is read only and then map it -fn verify_read_only_and_map(file_path: &str) -> Result { - let file = File::open(file_path)?; - let metadata = file.metadata()?; - ensure!( - metadata.permissions().readonly(), - "Cannot mmap file {} as it is not read only", - file_path - ); +fn verify_read_only_and_map(file_path: &str) -> Result { + let file = File::open(file_path) + .map_err(|errmsg| FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)))?; + let metadata = file.metadata().map_err(|errmsg| { + FileReadFail(anyhow!("Failed to find metadata for {}: {}", file_path, errmsg)) + })?; + + // ensure storage file is read only + if !metadata.permissions().readonly() { + return Err(MapFileFail(anyhow!("fail to map non read only storage file {}", file_path))); + } + // SAFETY: // // Mmap constructors are unsafe as it would have undefined behaviors if the file @@ -83,14 +99,19 @@ fn verify_read_only_and_map(file_path: &str) -> Result { // We should remove this restriction if we need to support mmap non read only file in // the future (by making this api unsafe). But for now, all flags are boot stable, so // the boot flag file copy should be readonly. - unsafe { Ok(Mmap::map(&file)?) } + unsafe { + let mapped_file = Mmap::map(&file).map_err(|errmsg| { + MapFileFail(anyhow!("fail to map storage file {}: {}", file_path, errmsg)) + })?; + Ok(mapped_file) + } } /// Map all storage files for a particular container fn map_container_storage_files( location_pb_file: &str, container: &str, -) -> Result { +) -> Result { let files_location = find_container_storage_location(location_pb_file, container)?; let package_map = Arc::new(verify_read_only_and_map(files_location.package_map())?); let flag_map = Arc::new(verify_read_only_and_map(files_location.flag_map())?); @@ -99,11 +120,11 @@ fn map_container_storage_files( } /// Get a mapped storage file given the container and file type -pub fn get_mapped_file( +pub(crate) fn get_mapped_file( location_pb_file: &str, container: &str, file_selection: StorageFileSelection, -) -> Result> { +) -> Result, AconfigStorageError> { let mut all_mapped_files = ALL_MAPPED_FILES.lock().unwrap(); match all_mapped_files.get(container) { Some(mapped_files) => Ok(match file_selection { @@ -127,7 +148,10 @@ pub fn get_mapped_file( #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{get_binary_storage_proto_bytes, write_bytes_to_temp_file}; + use crate::test_utils::{ + create_temp_storage_files_for_test, get_binary_storage_proto_bytes, + set_temp_storage_files_to_read_only, write_bytes_to_temp_file, + }; #[test] fn test_find_storage_file_location() { @@ -170,7 +194,10 @@ files { assert_eq!(file_info.timestamp(), 54321); let err = find_container_storage_location(&file_full_path, "vendor").unwrap_err(); - assert_eq!(format!("{:?}", err), "Storage file does not exist for vendor"); + assert_eq!( + format!("{:?}", err), + "StorageFileNotFound(Storage file does not exist for vendor)" + ); } fn map_and_verify( @@ -188,13 +215,17 @@ files { #[test] fn test_mapped_file_contents() { + #[cfg(feature = "cargo")] + create_temp_storage_files_for_test(); + + set_temp_storage_files_to_read_only(); let text_proto = r#" files { version: 0 container: "system" - package_map: "./tests/package.map" - flag_map: "./tests/flag.map" - flag_val: "./tests/flag.val" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.ro.flag.map" + flag_val: "./tests/tmp.ro.flag.val" timestamp: 12345 } "#; @@ -202,22 +233,28 @@ files { let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap(); let file_full_path = file.path().display().to_string(); - map_and_verify(&file_full_path, StorageFileSelection::PackageMap, "./tests/package.map"); - - map_and_verify(&file_full_path, StorageFileSelection::FlagMap, "./tests/flag.map"); - - map_and_verify(&file_full_path, StorageFileSelection::FlagVal, "./tests/flag.val"); + map_and_verify( + &file_full_path, + StorageFileSelection::PackageMap, + "./tests/tmp.ro.package.map", + ); + map_and_verify(&file_full_path, StorageFileSelection::FlagMap, "./tests/tmp.ro.flag.map"); + map_and_verify(&file_full_path, StorageFileSelection::FlagVal, "./tests/tmp.ro.flag.val"); } #[test] fn test_map_non_read_only_file() { + #[cfg(feature = "cargo")] + create_temp_storage_files_for_test(); + + set_temp_storage_files_to_read_only(); let text_proto = r#" files { version: 0 container: "system" - package_map: "./tests/rw.package.map" - flag_map: "./tests/rw.flag.map" - flag_val: "./tests/rw.flag.val" + package_map: "./tests/tmp.rw.package.map" + flag_map: "./tests/tmp.rw.flag.map" + flag_val: "./tests/tmp.rw.flag.val" timestamp: 12345 } "#; @@ -228,16 +265,16 @@ files { let error = map_container_storage_files(&file_full_path, "system").unwrap_err(); assert_eq!( format!("{:?}", error), - "Cannot mmap file ./tests/rw.package.map as it is not read only" + "MapFileFail(fail to map non read only storage file ./tests/tmp.rw.package.map)" ); let text_proto = r#" files { version: 0 container: "system" - package_map: "./tests/package.map" - flag_map: "./tests/rw.flag.map" - flag_val: "./tests/rw.flag.val" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.rw.flag.map" + flag_val: "./tests/tmp.rw.flag.val" timestamp: 12345 } "#; @@ -248,16 +285,16 @@ files { let error = map_container_storage_files(&file_full_path, "system").unwrap_err(); assert_eq!( format!("{:?}", error), - "Cannot mmap file ./tests/rw.flag.map as it is not read only" + "MapFileFail(fail to map non read only storage file ./tests/tmp.rw.flag.map)" ); let text_proto = r#" files { version: 0 container: "system" - package_map: "./tests/package.map" - flag_map: "./tests/flag.map" - flag_val: "./tests/rw.flag.val" + package_map: "./tests/tmp.ro.package.map" + flag_map: "./tests/tmp.ro.flag.map" + flag_val: "./tests/tmp.rw.flag.val" timestamp: 12345 } "#; @@ -268,7 +305,7 @@ files { let error = map_container_storage_files(&file_full_path, "system").unwrap_err(); assert_eq!( format!("{:?}", error), - "Cannot mmap file ./tests/rw.flag.val as it is not read only" + "MapFileFail(fail to map non read only storage file ./tests/tmp.rw.flag.val)" ); } } diff --git a/tools/aconfig/aconfig_storage_file/src/package_table.rs b/tools/aconfig/aconfig_storage_file/src/package_table.rs index a3ad6ec5b0..7308d7b3b2 100644 --- a/tools/aconfig/aconfig_storage_file/src/package_table.rs +++ b/tools/aconfig/aconfig_storage_file/src/package_table.rs @@ -17,8 +17,9 @@ //! package table module defines the package table file format and methods for serialization //! and deserialization +use crate::AconfigStorageError::{self, BytesParseFail, HigherStorageFileVersion}; use crate::{get_bucket_index, read_str_from_bytes, read_u32_from_bytes}; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; /// Package table header struct #[derive(PartialEq, Debug)] @@ -47,7 +48,7 @@ impl PackageTableHeader { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let mut head = 0; Ok(Self { version: read_u32_from_bytes(bytes, &mut head)?, @@ -85,7 +86,7 @@ impl PackageTableNode { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let mut head = 0; let node = Self { package_name: read_str_from_bytes(bytes, &mut head)?, @@ -127,7 +128,7 @@ impl PackageTable { } /// Deserialize from bytes - pub fn from_bytes(bytes: &[u8]) -> Result { + pub fn from_bytes(bytes: &[u8]) -> Result { let header = PackageTableHeader::from_bytes(bytes)?; let num_packages = header.num_packages; let num_buckets = crate::get_table_size(num_packages)?; @@ -144,7 +145,8 @@ impl PackageTable { head += node.as_bytes().len(); Ok(node) }) - .collect::>>()?; + .collect::, AconfigStorageError>>() + .map_err(|errmsg| BytesParseFail(anyhow!("fail to parse package table: {}", errmsg)))?; let table = Self { header, buckets, nodes }; Ok(table) @@ -159,18 +161,21 @@ pub struct PackageOffset { } /// Query package id and start offset -pub fn find_package_offset(buf: &[u8], package: &str) -> Result> { +pub fn find_package_offset( + buf: &[u8], + package: &str, +) -> Result, AconfigStorageError> { let interpreted_header = PackageTableHeader::from_bytes(buf)?; if interpreted_header.version > crate::FILE_VERSION { - return Err(anyhow!( + return Err(HigherStorageFileVersion(anyhow!( "Cannot read storage file with a higher version of {} with lib version {}", interpreted_header.version, crate::FILE_VERSION - )); + ))); } let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4; - let bucket_index = PackageTableNode::find_bucket_index(&package, num_buckets); + let bucket_index = PackageTableNode::find_bucket_index(package, num_buckets); let mut pos = (interpreted_header.bucket_offset + 4 * bucket_index) as usize; let mut package_node_offset = read_u32_from_bytes(buf, &mut pos)? as usize; @@ -199,7 +204,7 @@ pub fn find_package_offset(buf: &[u8], package: &str) -> Result Result { + pub fn create_test_package_table() -> PackageTable { let header = PackageTableHeader { version: crate::FILE_VERSION, container: String::from("system"), @@ -228,14 +233,13 @@ mod tests { next_offset: None, }; let nodes = vec![first_node, second_node, third_node]; - Ok(PackageTable { header, buckets, nodes }) + PackageTable { header, buckets, nodes } } #[test] // this test point locks down the table serialization fn test_serialization() { - let package_table = create_test_package_table().unwrap(); - + let package_table = create_test_package_table(); let header: &PackageTableHeader = &package_table.header; let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes()); assert!(reinterpreted_header.is_ok()); @@ -255,7 +259,7 @@ mod tests { #[test] // this test point locks down table query fn test_package_query() { - let package_table = create_test_package_table().unwrap().as_bytes(); + let package_table = create_test_package_table().as_bytes(); let package_offset = find_package_offset(&package_table[..], "com.android.aconfig.storage.test_1") .unwrap() @@ -280,7 +284,7 @@ mod tests { // this test point locks down table query of a non exist package fn test_not_existed_package_query() { // this will land at an empty bucket - let package_table = create_test_package_table().unwrap().as_bytes(); + let package_table = create_test_package_table().as_bytes(); let package_offset = find_package_offset(&package_table[..], "com.android.aconfig.storage.test_3").unwrap(); assert_eq!(package_offset, None); @@ -293,7 +297,7 @@ mod tests { #[test] // this test point locks down query error when file has a higher version fn test_higher_version_storage_file() { - let mut table = create_test_package_table().unwrap(); + let mut table = create_test_package_table(); table.header.version = crate::FILE_VERSION + 1; let package_table = table.as_bytes(); let error = find_package_offset(&package_table[..], "com.android.aconfig.storage.test_1") @@ -301,7 +305,7 @@ mod tests { assert_eq!( format!("{:?}", error), format!( - "Cannot read storage file with a higher version of {} with lib version {}", + "HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})", crate::FILE_VERSION + 1, crate::FILE_VERSION ) diff --git a/tools/aconfig/aconfig_storage_file/src/test_utils.rs b/tools/aconfig/aconfig_storage_file/src/test_utils.rs index c468683407..6fe5a279a4 100644 --- a/tools/aconfig/aconfig_storage_file/src/test_utils.rs +++ b/tools/aconfig/aconfig_storage_file/src/test_utils.rs @@ -17,18 +17,74 @@ use crate::protos::ProtoStorageFiles; use anyhow::Result; use protobuf::Message; +use std::fs; use std::io::Write; +use std::path::Path; +use std::sync::Once; use tempfile::NamedTempFile; -pub fn get_binary_storage_proto_bytes(text_proto: &str) -> Result> { +static INIT: Once = Once::new(); + +pub(crate) fn get_binary_storage_proto_bytes(text_proto: &str) -> Result> { let storage_files: ProtoStorageFiles = protobuf::text_format::parse_from_str(text_proto)?; let mut binary_proto = Vec::new(); storage_files.write_to_vec(&mut binary_proto)?; Ok(binary_proto) } -pub fn write_bytes_to_temp_file(bytes: &[u8]) -> Result { +pub(crate) fn write_bytes_to_temp_file(bytes: &[u8]) -> Result { let mut file = NamedTempFile::new()?; let _ = file.write_all(&bytes); Ok(file) } + +fn has_same_content(file1: &Path, file2: &Path) -> Result { + let bytes1 = fs::read(file1)?; + let bytes2 = fs::read(file2)?; + if bytes1.len() != bytes2.len() { + return Ok(false); + } + for (i, &b1) in bytes1.iter().enumerate() { + if b1 != bytes2[i] { + return Ok(false); + } + } + Ok(true) +} + +pub(crate) fn create_temp_storage_files_for_test() { + INIT.call_once(|| { + let file_paths = [ + ("./tests/package.map", "./tests/tmp.ro.package.map"), + ("./tests/flag.map", "./tests/tmp.ro.flag.map"), + ("./tests/flag.val", "./tests/tmp.ro.flag.val"), + ("./tests/package.map", "./tests/tmp.rw.package.map"), + ("./tests/flag.map", "./tests/tmp.rw.flag.map"), + ("./tests/flag.val", "./tests/tmp.rw.flag.val"), + ]; + for (file_path, copied_file_path) in file_paths.into_iter() { + let file_path = Path::new(&file_path); + let copied_file_path = Path::new(&copied_file_path); + if copied_file_path.exists() && !has_same_content(file_path, copied_file_path).unwrap() + { + fs::remove_file(copied_file_path).unwrap(); + } + if !copied_file_path.exists() { + fs::copy(file_path, copied_file_path).unwrap(); + } + } + }); +} + +pub(crate) fn set_temp_storage_files_to_read_only() { + let file_paths = + ["./tests/tmp.ro.package.map", "./tests/tmp.ro.flag.map", "./tests/tmp.ro.flag.val"]; + for file_path in file_paths.into_iter() { + let file_path = Path::new(&file_path); + let mut perms = fs::metadata(file_path).unwrap().permissions(); + if !perms.readonly() { + perms.set_readonly(true); + fs::set_permissions(file_path, perms).unwrap(); + } + } +} diff --git a/tools/aconfig/aconfig_storage_file/tests/rw.flag.map b/tools/aconfig/aconfig_storage_file/tests/rw.flag.map deleted file mode 100644 index 43b6f9a640..0000000000 Binary files a/tools/aconfig/aconfig_storage_file/tests/rw.flag.map and /dev/null differ diff --git a/tools/aconfig/aconfig_storage_file/tests/rw.flag.val b/tools/aconfig/aconfig_storage_file/tests/rw.flag.val deleted file mode 100644 index f39f8d3aaf..0000000000 Binary files a/tools/aconfig/aconfig_storage_file/tests/rw.flag.val and /dev/null differ diff --git a/tools/aconfig/aconfig_storage_file/tests/rw.package.map b/tools/aconfig/aconfig_storage_file/tests/rw.package.map deleted file mode 100644 index 8ed4767a56..0000000000 Binary files a/tools/aconfig/aconfig_storage_file/tests/rw.package.map and /dev/null differ