tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "aconfig: add top level flag read lib api" into main

Browse source
This commit is contained in:
Dennis Shen 2024-02-08 18:17:27 +00:00 committed by Gerrit Code Review
commit e7fde3a86f
11 changed files with 551 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
tools/aconfig/aconfig_storage_file/Android.bp
View file

@ -13,6 +13,9 @@ rust_defaults {
"libonce_cell",
"libprotobuf",
"libtempfile",
"libmemmap2",
"libcxx",
"libthiserror",
],
}
@ -23,10 +26,60 @@ rust_library {
defaults: ["aconfig_storage_file.defaults"],
}
genrule {
name: "ro.package.map",
out: ["tests/tmp.ro.package.map"],
srcs: ["tests/package.map"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)",
}
genrule {
name: "ro.flag.map",
out: ["tests/tmp.ro.flag.map"],
srcs: ["tests/flag.map"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)",
}
genrule {
name: "ro.flag.val",
out: ["tests/tmp.ro.flag.val"],
srcs: ["tests/flag.val"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod -w $(out)",
}
genrule {
name: "rw.package.map",
out: ["tests/tmp.rw.package.map"],
srcs: ["tests/package.map"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)",
}
genrule {
name: "rw.flag.map",
out: ["tests/tmp.rw.flag.map"],
srcs: ["tests/flag.map"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)",
}
genrule {
name: "rw.flag.val",
out: ["tests/tmp.rw.flag.val"],
srcs: ["tests/flag.val"],
cmd: "rm -f $(out);cp -f $(in) $(out);chmod +w $(out)",
}
rust_test_host {
name: "aconfig_storage_file.test",
test_suites: ["general-tests"],
defaults: ["aconfig_storage_file.defaults"],
data: [
":ro.package.map",
":ro.flag.map",
":ro.flag.val",
":rw.package.map",
":rw.flag.map",
":rw.flag.val",
],
}
rust_protobuf {

2
tools/aconfig/aconfig_storage_file/Cargo.toml
View file

@ -13,6 +13,8 @@ memmap2 = "0.8.0"
protobuf = "3.2.0"
once_cell = "1.19.0"
tempfile = "3.9.0"
cxx = "1.0"
thiserror = "1.0.56"
[build-dependencies]
protobuf-codegen = "3.2.0"

37
tools/aconfig/aconfig_storage_file/src/flag_table.rs
View file

@ -17,8 +17,10 @@
//! flag table module defines the flag table file format and methods for serialization
//! and deserialization
use crate::AconfigStorageError::{self, BytesParseFail, HigherStorageFileVersion};
use crate::{get_bucket_index, read_str_from_bytes, read_u16_from_bytes, read_u32_from_bytes};
use anyhow::{anyhow, Result};
use anyhow::anyhow;
pub type FlagOffset = u16;
/// Flag table header struct
#[derive(PartialEq, Debug)]
@ -47,7 +49,7 @@ impl FlagTableHeader {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let mut head = 0;
Ok(Self {
version: read_u32_from_bytes(bytes, &mut head)?,
@ -85,7 +87,7 @@ impl FlagTableNode {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let mut head = 0;
let node = Self {
package_id: read_u32_from_bytes(bytes, &mut head)?,
@ -127,7 +129,7 @@ impl FlagTable {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let header = FlagTableHeader::from_bytes(bytes)?;
let num_flags = header.num_flags;
let num_buckets = crate::get_table_size(num_flags)?;
@ -144,7 +146,8 @@ impl FlagTable {
head += node.as_bytes().len();
Ok(node)
})
.collect::<Result<Vec<_>>>()?;
.collect::<Result<Vec<_>, AconfigStorageError>>()
.map_err(|errmsg| BytesParseFail(anyhow!("fail to parse flag table: {}", errmsg)))?;
let table = Self { header, buckets, nodes };
Ok(table)
@ -152,14 +155,18 @@ impl FlagTable {
}
/// Query flag within package offset
pub fn find_flag_offset(buf: &[u8], package_id: u32, flag: &str) -> Result<Option<u16>> {
pub fn find_flag_offset(
buf: &[u8],
package_id: u32,
flag: &str,
) -> Result<Option<FlagOffset>, AconfigStorageError> {
let interpreted_header = FlagTableHeader::from_bytes(buf)?;
if interpreted_header.version > crate::FILE_VERSION {
return Err(anyhow!(
return Err(HigherStorageFileVersion(anyhow!(
"Cannot read storage file with a higher version of {} with lib version {}",
interpreted_header.version,
crate::FILE_VERSION
));
)));
}
let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4;
@ -202,7 +209,7 @@ mod tests {
}
}
pub fn create_test_flag_table() -> Result<FlagTable> {
pub fn create_test_flag_table() -> FlagTable {
let header = FlagTableHeader {
version: crate::FILE_VERSION,
container: String::from("system"),
@ -240,13 +247,13 @@ mod tests {
FlagTableNode::new_expected(2, "enabled_fixed_ro", 1, 0, None),
FlagTableNode::new_expected(0, "disabled_rw", 1, 0, None),
];
Ok(FlagTable { header, buckets, nodes })
FlagTable { header, buckets, nodes }
}
#[test]
// this test point locks down the table serialization
fn test_serialization() {
let flag_table = create_test_flag_table().unwrap();
let flag_table = create_test_flag_table();
let header: &FlagTableHeader = &flag_table.header;
let reinterpreted_header = FlagTableHeader::from_bytes(&header.as_bytes());
@ -267,7 +274,7 @@ mod tests {
#[test]
// this test point locks down table query
fn test_flag_query() {
let flag_table = create_test_flag_table().unwrap().as_bytes();
let flag_table = create_test_flag_table().as_bytes();
let baseline = vec![
(0, "enabled_ro", 1u16),
(0, "enabled_rw", 2u16),
@ -288,7 +295,7 @@ mod tests {
#[test]
// this test point locks down table query of a non exist flag
fn test_not_existed_flag_query() {
let flag_table = create_test_flag_table().unwrap().as_bytes();
let flag_table = create_test_flag_table().as_bytes();
let flag_offset = find_flag_offset(&flag_table[..], 1, "disabled_fixed_ro").unwrap();
assert_eq!(flag_offset, None);
let flag_offset = find_flag_offset(&flag_table[..], 2, "disabled_rw").unwrap();
@ -298,14 +305,14 @@ mod tests {
#[test]
// this test point locks down query error when file has a higher version
fn test_higher_version_storage_file() {
let mut table = create_test_flag_table().unwrap();
let mut table = create_test_flag_table();
table.header.version = crate::FILE_VERSION + 1;
let flag_table = table.as_bytes();
let error = find_flag_offset(&flag_table[..], 0, "enabled_ro").unwrap_err();
assert_eq!(
format!("{:?}", error),
format!(
"Cannot read storage file with a higher version of {} with lib version {}",
"HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})",
crate::FILE_VERSION + 1,
crate::FILE_VERSION
)

42
tools/aconfig/aconfig_storage_file/src/flag_value.rs
View file

@ -17,8 +17,9 @@
//! flag value module defines the flag value file format and methods for serialization
//! and deserialization
use crate::AconfigStorageError::{self, HigherStorageFileVersion, InvalidStorageFileOffset};
use crate::{read_str_from_bytes, read_u32_from_bytes, read_u8_from_bytes};
use anyhow::{anyhow, Result};
use anyhow::anyhow;
/// Flag value header struct
#[derive(PartialEq, Debug)]
@ -45,7 +46,7 @@ impl FlagValueHeader {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let mut head = 0;
Ok(Self {
version: read_u32_from_bytes(bytes, &mut head)?,
@ -75,7 +76,7 @@ impl FlagValueList {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let header = FlagValueHeader::from_bytes(bytes)?;
let num_flags = header.num_flags;
let mut head = header.as_bytes().len();
@ -87,14 +88,14 @@ impl FlagValueList {
}
/// Query flag value
pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result<bool> {
pub fn find_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result<bool, AconfigStorageError> {
let interpreted_header = FlagValueHeader::from_bytes(buf)?;
if interpreted_header.version > crate::FILE_VERSION {
return Err(anyhow!(
return Err(HigherStorageFileVersion(anyhow!(
"Cannot read storage file with a higher version of {} with lib version {}",
interpreted_header.version,
crate::FILE_VERSION
));
)));
}
let mut head = (interpreted_header.boolean_value_offset + flag_offset) as usize;
@ -102,7 +103,9 @@ pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result<bool> {
// TODO: right now, there is only boolean flags, with more flag value types added
// later, the end of boolean flag value section should be updated (b/322826265).
if head >= interpreted_header.file_size as usize {
return Err(anyhow!("Flag value offset goes beyond the end of the file."));
return Err(InvalidStorageFileOffset(anyhow!(
"Flag value offset goes beyond the end of the file."
)));
}
let val = read_u8_from_bytes(buf, &mut head)?;
@ -113,7 +116,7 @@ pub fn get_boolean_flag_value(buf: &[u8], flag_offset: u32) -> Result<bool> {
mod tests {
use super::*;
pub fn create_test_flag_value_list() -> Result<FlagValueList> {
pub fn create_test_flag_value_list() -> FlagValueList {
let header = FlagValueHeader {
version: crate::FILE_VERSION,
container: String::from("system"),
@ -122,13 +125,13 @@ mod tests {
boolean_value_offset: 26,
};
let booleans: Vec<bool> = vec![false, true, false, false, true, true, false, true];
Ok(FlagValueList { header, booleans })
FlagValueList { header, booleans }
}
#[test]
// this test point locks down the value list serialization
fn test_serialization() {
let flag_value_list = create_test_flag_value_list().unwrap();
let flag_value_list = create_test_flag_value_list();
let header: &FlagValueHeader = &flag_value_list.header;
let reinterpreted_header = FlagValueHeader::from_bytes(&header.as_bytes());
@ -143,10 +146,10 @@ mod tests {
#[test]
// this test point locks down flag value query
fn test_flag_value_query() {
let flag_value_list = create_test_flag_value_list().unwrap().as_bytes();
let flag_value_list = create_test_flag_value_list().as_bytes();
let baseline: Vec<bool> = vec![false, true, false, false, true, true, false, true];
for (offset, expected_value) in baseline.into_iter().enumerate() {
let flag_value = get_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap();
let flag_value = find_boolean_flag_value(&flag_value_list[..], offset as u32).unwrap();
assert_eq!(flag_value, expected_value);
}
}
@ -154,22 +157,25 @@ mod tests {
#[test]
// this test point locks down query beyond the end of boolean section
fn test_boolean_out_of_range() {
let flag_value_list = create_test_flag_value_list().unwrap().as_bytes();
let error = get_boolean_flag_value(&flag_value_list[..], 8).unwrap_err();
assert_eq!(format!("{:?}", error), "Flag value offset goes beyond the end of the file.");
let flag_value_list = create_test_flag_value_list().as_bytes();
let error = find_boolean_flag_value(&flag_value_list[..], 8).unwrap_err();
assert_eq!(
format!("{:?}", error),
"InvalidStorageFileOffset(Flag value offset goes beyond the end of the file.)"
);
}
#[test]
// this test point locks down query error when file has a higher version
fn test_higher_version_storage_file() {
let mut value_list = create_test_flag_value_list().unwrap();
let mut value_list = create_test_flag_value_list();
value_list.header.version = crate::FILE_VERSION + 1;
let flag_value = value_list.as_bytes();
let error = get_boolean_flag_value(&flag_value[..], 4).unwrap_err();
let error = find_boolean_flag_value(&flag_value[..], 4).unwrap_err();
assert_eq!(
format!("{:?}", error),
format!(
"Cannot read storage file with a higher version of {} with lib version {}",
"HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})",
crate::FILE_VERSION + 1,
crate::FILE_VERSION
)

313
tools/aconfig/aconfig_storage_file/src/lib.rs
View file

@ -15,37 +15,56 @@
*/
//! `aconfig_storage_file` is a crate that defines aconfig storage file format, it
//! also includes apis to read flags from storage files
//! also includes apis to read flags from storage files. It provides three apis to
//! interface with storage files:
//!
//! 1, function to get package flag value start offset
//! pub fn get_package_offset(container: &str, package: &str) -> `Result<Option<PackageOffset>>>`
//!
//! 2, function to get flag offset within a specific package
//! pub fn get_flag_offset(container: &str, package_id: u32, flag: &str) -> `Result<Option<u16>>>`
//!
//! 3, function to get the actual flag value given the global offset (combined package and
//! flag offset).
//! pub fn get_boolean_flag_value(container: &str, offset: u32) -> `Result<bool>`
//!
//! Note these are low level apis that are expected to be only used in auto generated flag
//! apis. DO NOT DIRECTLY USE THESE APIS IN YOUR SOURCE CODE. For auto generated flag apis
//! please refer to the g3doc go/android-flags
pub mod flag_table;
pub mod flag_value;
pub mod package_table;
#[cfg(feature = "cargo")]
pub mod mapped_file;
pub mod package_table;
pub mod protos;
mod protos;
#[cfg(test)]
mod test_utils;
use anyhow::{anyhow, Result};
use anyhow::anyhow;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode};
pub use crate::flag_table::{FlagOffset, FlagTable, FlagTableHeader, FlagTableNode};
pub use crate::flag_value::{FlagValueHeader, FlagValueList};
pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode};
pub use crate::package_table::{PackageOffset, PackageTable, PackageTableHeader, PackageTableNode};
pub use crate::protos::ProtoStorageFiles;
use crate::AconfigStorageError::{BytesParseFail, HashTableSizeLimit};
/// Storage file version
pub const FILE_VERSION: u32 = 1;
/// Good hash table prime number
pub const HASH_PRIMES: [u32; 29] = [
pub(crate) const HASH_PRIMES: [u32; 29] = [
7, 17, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241,
786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611,
402653189, 805306457, 1610612741,
];
/// Storage file location pb file
pub const STORAGE_LOCATION_FILE: &str = "/metadata/aconfig/storage_files.pb";
/// Storage file type enum
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum StorageFileSelection {
@ -69,46 +88,300 @@ impl TryFrom<&str> for StorageFileSelection {
/// Get the right hash table size given number of entries in the table. Use a
/// load factor of 0.5 for performance.
pub fn get_table_size(entries: u32) -> Result<u32> {
pub fn get_table_size(entries: u32) -> Result<u32, AconfigStorageError> {
HASH_PRIMES
.iter()
.find(|&&num| num >= 2 * entries)
.copied()
.ok_or(anyhow!("Number of packages is too large"))
.ok_or(HashTableSizeLimit(anyhow!("Number of items in a hash table exceeds limit")))
}
/// Get the corresponding bucket index given the key and number of buckets
pub fn get_bucket_index<T: Hash>(val: &T, num_buckets: u32) -> u32 {
pub(crate) fn get_bucket_index<T: Hash>(val: &T, num_buckets: u32) -> u32 {
let mut s = DefaultHasher::new();
val.hash(&mut s);
(s.finish() % num_buckets as u64) as u32
}
/// Read and parse bytes as u8
pub fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result<u8> {
let val = u8::from_le_bytes(buf[*head..*head + 1].try_into()?);
pub(crate) fn read_u8_from_bytes(buf: &[u8], head: &mut usize) -> Result<u8, AconfigStorageError> {
let val =
u8::from_le_bytes(buf[*head..*head + 1].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u8 from bytes: {}", errmsg))
})?);
*head += 1;
Ok(val)
}
/// Read and parse bytes as u16
pub fn read_u16_from_bytes(buf: &[u8], head: &mut usize) -> Result<u16> {
let val = u16::from_le_bytes(buf[*head..*head + 2].try_into()?);
pub(crate) fn read_u16_from_bytes(
buf: &[u8],
head: &mut usize,
) -> Result<u16, AconfigStorageError> {
let val =
u16::from_le_bytes(buf[*head..*head + 2].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u16 from bytes: {}", errmsg))
})?);
*head += 2;
Ok(val)
}
/// Read and parse bytes as u32
pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32> {
let val = u32::from_le_bytes(buf[*head..*head + 4].try_into()?);
pub(crate) fn read_u32_from_bytes(
buf: &[u8],
head: &mut usize,
) -> Result<u32, AconfigStorageError> {
let val =
u32::from_le_bytes(buf[*head..*head + 4].try_into().map_err(|errmsg| {
BytesParseFail(anyhow!("fail to parse u32 from bytes: {}", errmsg))
})?);
*head += 4;
Ok(val)
}
/// Read and parse bytes as string
pub fn read_str_from_bytes(buf: &[u8], head: &mut usize) -> Result<String> {
pub(crate) fn read_str_from_bytes(
buf: &[u8],
head: &mut usize,
) -> Result<String, AconfigStorageError> {
let num_bytes = read_u32_from_bytes(buf, head)? as usize;
let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())?;
let val = String::from_utf8(buf[*head..*head + num_bytes].to_vec())
.map_err(|errmsg| BytesParseFail(anyhow!("fail to parse string from bytes: {}", errmsg)))?;
*head += num_bytes;
Ok(val)
}
/// Storage query api error
#[non_exhaustive]
#[derive(thiserror::Error, Debug)]
pub enum AconfigStorageError {
#[error("failed to read the file")]
FileReadFail(#[source] anyhow::Error),
#[error("fail to parse protobuf")]
ProtobufParseFail(#[source] anyhow::Error),
#[error("storage files not found for this container")]
StorageFileNotFound(#[source] anyhow::Error),
#[error("fail to map storage file")]
MapFileFail(#[source] anyhow::Error),
#[error("number of items in hash table exceed limit")]
HashTableSizeLimit(#[source] anyhow::Error),
#[error("failed to parse bytes into data")]
BytesParseFail(#[source] anyhow::Error),
#[error("cannot parse storage files with a higher version")]
HigherStorageFileVersion(#[source] anyhow::Error),
#[error("invalid storage file byte offset")]
InvalidStorageFileOffset(#[source] anyhow::Error),
}
/// Get package start offset implementation
pub fn get_package_offset_impl(
pb_file: &str,
container: &str,
package: &str,
) -> Result<Option<PackageOffset>, AconfigStorageError> {
let mapped_file =
crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::PackageMap)?;
crate::package_table::find_package_offset(&mapped_file, package)
}
/// Get flag offset implementation
pub fn get_flag_offset_impl(
pb_file: &str,
container: &str,
package_id: u32,
flag: &str,
) -> Result<Option<FlagOffset>, AconfigStorageError> {
let mapped_file =
crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::FlagMap)?;
crate::flag_table::find_flag_offset(&mapped_file, package_id, flag)
}
/// Get boolean flag value implementation
pub fn get_boolean_flag_value_impl(
pb_file: &str,
container: &str,
offset: u32,
) -> Result<bool, AconfigStorageError> {
let mapped_file =
crate::mapped_file::get_mapped_file(pb_file, container, StorageFileSelection::FlagVal)?;
crate::flag_value::find_boolean_flag_value(&mapped_file, offset)
}
/// Get package start offset for flags given the container and package name.
///
/// This function would map the corresponding package map file if has not been mapped yet,
/// and then look for the target package in this mapped file.
///
/// If a package is found, it returns Ok(Some(PackageOffset))
/// If a package is not found, it returns Ok(None)
/// If errors out such as no such package map file is found, it returns an Err(errmsg)
pub fn get_package_offset(
container: &str,
package: &str,
) -> Result<Option<PackageOffset>, AconfigStorageError> {
get_package_offset_impl(STORAGE_LOCATION_FILE, container, package)
}
/// Get flag offset within a package given the container name, package id and flag name.
///
/// This function would map the corresponding flag map file if has not been mapped yet,
/// and then look for the target flag in this mapped file.
///
/// If a flag is found, it returns Ok(Some(u16))
/// If a flag is not found, it returns Ok(None)
/// If errors out such as no such flag map file is found, it returns an Err(errmsg)
pub fn get_flag_offset(
container: &str,
package_id: u32,
flag: &str,
) -> Result<Option<FlagOffset>, AconfigStorageError> {
get_flag_offset_impl(STORAGE_LOCATION_FILE, container, package_id, flag)
}
/// Get the boolean flag value given the container name and flag global offset
///
/// This function would map the corresponding flag value file if has not been mapped yet,
/// and then look for the target flag value at the specified offset.
///
/// If flag value file is successfully mapped and the provide offset is valid, it returns
/// the boolean flag value, otherwise it returns the error message.
pub fn get_boolean_flag_value(container: &str, offset: u32) -> Result<bool, AconfigStorageError> {
get_boolean_flag_value_impl(STORAGE_LOCATION_FILE, container, offset)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::{
create_temp_storage_files_for_test, get_binary_storage_proto_bytes,
set_temp_storage_files_to_read_only, write_bytes_to_temp_file,
};
#[test]
// this test point locks down flag package offset query
fn test_package_offset_query() {
#[cfg(feature = "cargo")]
create_temp_storage_files_for_test();
set_temp_storage_files_to_read_only();
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.ro.flag.map"
flag_val: "./tests/tmp.ro.flag.val"
timestamp: 12345
}
"#;
let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap();
let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap();
let file_full_path = file.path().display().to_string();
let package_offset = get_package_offset_impl(
&file_full_path,
"system",
"com.android.aconfig.storage.test_1",
)
.unwrap()
.unwrap();
let expected_package_offset = PackageOffset { package_id: 0, boolean_offset: 0 };
assert_eq!(package_offset, expected_package_offset);
let package_offset = get_package_offset_impl(
&file_full_path,
"system",
"com.android.aconfig.storage.test_2",
)
.unwrap()
.unwrap();
let expected_package_offset = PackageOffset { package_id: 1, boolean_offset: 3 };
assert_eq!(package_offset, expected_package_offset);
let package_offset = get_package_offset_impl(
&file_full_path,
"system",
"com.android.aconfig.storage.test_4",
)
.unwrap()
.unwrap();
let expected_package_offset = PackageOffset { package_id: 2, boolean_offset: 6 };
assert_eq!(package_offset, expected_package_offset);
}
#[test]
// this test point locks down flag offset query
fn test_flag_offset_query() {
#[cfg(feature = "cargo")]
create_temp_storage_files_for_test();
set_temp_storage_files_to_read_only();
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.ro.flag.map"
flag_val: "./tests/tmp.ro.flag.val"
timestamp: 12345
}
"#;
let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap();
let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap();
let file_full_path = file.path().display().to_string();
let baseline = vec![
(0, "enabled_ro", 1u16),
(0, "enabled_rw", 2u16),
(1, "disabled_ro", 0u16),
(2, "enabled_ro", 1u16),
(1, "enabled_fixed_ro", 1u16),
(1, "enabled_ro", 2u16),
(2, "enabled_fixed_ro", 0u16),
(0, "disabled_rw", 0u16),
];
for (package_id, flag_name, expected_offset) in baseline.into_iter() {
let flag_offset =
get_flag_offset_impl(&file_full_path, "system", package_id, flag_name)
.unwrap()
.unwrap();
assert_eq!(flag_offset, expected_offset);
}
}
#[test]
// this test point locks down flag offset query
fn test_flag_value_query() {
#[cfg(feature = "cargo")]
create_temp_storage_files_for_test();
set_temp_storage_files_to_read_only();
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.ro.flag.map"
flag_val: "./tests/tmp.ro.flag.val"
timestamp: 12345
}
"#;
let binary_proto_bytes = get_binary_storage_proto_bytes(text_proto).unwrap();
let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap();
let file_full_path = file.path().display().to_string();
let baseline: Vec<bool> = vec![false; 8];
for (offset, expected_value) in baseline.into_iter().enumerate() {
let flag_value =
get_boolean_flag_value_impl(&file_full_path, "system", offset as u32).unwrap();
assert_eq!(flag_value, expected_value);
}
}
}

119
tools/aconfig/aconfig_storage_file/src/mapped_file.rs
View file

@ -19,13 +19,16 @@ use std::fs::File;
use std::io::{BufReader, Read};
use std::sync::{Arc, Mutex};
use anyhow::{bail, ensure, Result};
use anyhow::anyhow;
use memmap2::Mmap;
use once_cell::sync::Lazy;
use crate::protos::{
storage_files::try_from_binary_proto, ProtoStorageFileInfo, ProtoStorageFiles,
};
use crate::AconfigStorageError::{
self, FileReadFail, MapFileFail, ProtobufParseFail, StorageFileNotFound,
};
use crate::StorageFileSelection;
/// Cache for already mapped files
@ -46,30 +49,43 @@ struct MappedStorageFileSet {
fn find_container_storage_location(
location_pb_file: &str,
container: &str,
) -> Result<ProtoStorageFileInfo> {
let file = File::open(location_pb_file)?;
) -> Result<ProtoStorageFileInfo, AconfigStorageError> {
let file = File::open(location_pb_file).map_err(|errmsg| {
FileReadFail(anyhow!("Failed to open file {}: {}", location_pb_file, errmsg))
})?;
let mut reader = BufReader::new(file);
let mut bytes = Vec::new();
reader.read_to_end(&mut bytes)?;
let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes)?;
reader.read_to_end(&mut bytes).map_err(|errmsg| {
FileReadFail(anyhow!("Failed to read file {}: {}", location_pb_file, errmsg))
})?;
let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes).map_err(|errmsg| {
ProtobufParseFail(anyhow!(
"Failed to parse storage location pb file {}: {}",
location_pb_file,
errmsg
))
})?;
for location_info in storage_locations.files.iter() {
if location_info.container() == container {
return Ok(location_info.clone());
}
}
bail!("Storage file does not exist for {}", container)
Err(StorageFileNotFound(anyhow!("Storage file does not exist for {}", container)))
}
/// Verify the file is read only and then map it
fn verify_read_only_and_map(file_path: &str) -> Result<Mmap> {
let file = File::open(file_path)?;
let metadata = file.metadata()?;
ensure!(
metadata.permissions().readonly(),
"Cannot mmap file {} as it is not read only",
file_path
);
fn verify_read_only_and_map(file_path: &str) -> Result<Mmap, AconfigStorageError> {
let file = File::open(file_path)
.map_err(|errmsg| FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)))?;
let metadata = file.metadata().map_err(|errmsg| {
FileReadFail(anyhow!("Failed to find metadata for {}: {}", file_path, errmsg))
})?;
// ensure storage file is read only
if !metadata.permissions().readonly() {
return Err(MapFileFail(anyhow!("fail to map non read only storage file {}", file_path)));
}
// SAFETY:
//
// Mmap constructors are unsafe as it would have undefined behaviors if the file
@ -83,14 +99,19 @@ fn verify_read_only_and_map(file_path: &str) -> Result<Mmap> {
// We should remove this restriction if we need to support mmap non read only file in
// the future (by making this api unsafe). But for now, all flags are boot stable, so
// the boot flag file copy should be readonly.
unsafe { Ok(Mmap::map(&file)?) }
unsafe {
let mapped_file = Mmap::map(&file).map_err(|errmsg| {
MapFileFail(anyhow!("fail to map storage file {}: {}", file_path, errmsg))
})?;
Ok(mapped_file)
}
}
/// Map all storage files for a particular container
fn map_container_storage_files(
location_pb_file: &str,
container: &str,
) -> Result<MappedStorageFileSet> {
) -> Result<MappedStorageFileSet, AconfigStorageError> {
let files_location = find_container_storage_location(location_pb_file, container)?;
let package_map = Arc::new(verify_read_only_and_map(files_location.package_map())?);
let flag_map = Arc::new(verify_read_only_and_map(files_location.flag_map())?);
@ -99,11 +120,11 @@ fn map_container_storage_files(
}
/// Get a mapped storage file given the container and file type
pub fn get_mapped_file(
pub(crate) fn get_mapped_file(
location_pb_file: &str,
container: &str,
file_selection: StorageFileSelection,
) -> Result<Arc<Mmap>> {
) -> Result<Arc<Mmap>, AconfigStorageError> {
let mut all_mapped_files = ALL_MAPPED_FILES.lock().unwrap();
match all_mapped_files.get(container) {
Some(mapped_files) => Ok(match file_selection {
@ -127,7 +148,10 @@ pub fn get_mapped_file(
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::{get_binary_storage_proto_bytes, write_bytes_to_temp_file};
use crate::test_utils::{
create_temp_storage_files_for_test, get_binary_storage_proto_bytes,
set_temp_storage_files_to_read_only, write_bytes_to_temp_file,
};
#[test]
fn test_find_storage_file_location() {
@ -170,7 +194,10 @@ files {
assert_eq!(file_info.timestamp(), 54321);
let err = find_container_storage_location(&file_full_path, "vendor").unwrap_err();
assert_eq!(format!("{:?}", err), "Storage file does not exist for vendor");
assert_eq!(
format!("{:?}", err),
"StorageFileNotFound(Storage file does not exist for vendor)"
);
}
fn map_and_verify(
@ -188,13 +215,17 @@ files {
#[test]
fn test_mapped_file_contents() {
#[cfg(feature = "cargo")]
create_temp_storage_files_for_test();
set_temp_storage_files_to_read_only();
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/package.map"
flag_map: "./tests/flag.map"
flag_val: "./tests/flag.val"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.ro.flag.map"
flag_val: "./tests/tmp.ro.flag.val"
timestamp: 12345
}
"#;
@ -202,22 +233,28 @@ files {
let file = write_bytes_to_temp_file(&binary_proto_bytes).unwrap();
let file_full_path = file.path().display().to_string();
map_and_verify(&file_full_path, StorageFileSelection::PackageMap, "./tests/package.map");
map_and_verify(&file_full_path, StorageFileSelection::FlagMap, "./tests/flag.map");
map_and_verify(&file_full_path, StorageFileSelection::FlagVal, "./tests/flag.val");
map_and_verify(
&file_full_path,
StorageFileSelection::PackageMap,
"./tests/tmp.ro.package.map",
);
map_and_verify(&file_full_path, StorageFileSelection::FlagMap, "./tests/tmp.ro.flag.map");
map_and_verify(&file_full_path, StorageFileSelection::FlagVal, "./tests/tmp.ro.flag.val");
}
#[test]
fn test_map_non_read_only_file() {
#[cfg(feature = "cargo")]
create_temp_storage_files_for_test();
set_temp_storage_files_to_read_only();
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/rw.package.map"
flag_map: "./tests/rw.flag.map"
flag_val: "./tests/rw.flag.val"
package_map: "./tests/tmp.rw.package.map"
flag_map: "./tests/tmp.rw.flag.map"
flag_val: "./tests/tmp.rw.flag.val"
timestamp: 12345
}
"#;
@ -228,16 +265,16 @@ files {
let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
assert_eq!(
format!("{:?}", error),
"Cannot mmap file ./tests/rw.package.map as it is not read only"
"MapFileFail(fail to map non read only storage file ./tests/tmp.rw.package.map)"
);
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/package.map"
flag_map: "./tests/rw.flag.map"
flag_val: "./tests/rw.flag.val"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.rw.flag.map"
flag_val: "./tests/tmp.rw.flag.val"
timestamp: 12345
}
"#;
@ -248,16 +285,16 @@ files {
let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
assert_eq!(
format!("{:?}", error),
"Cannot mmap file ./tests/rw.flag.map as it is not read only"
"MapFileFail(fail to map non read only storage file ./tests/tmp.rw.flag.map)"
);
let text_proto = r#"
files {
version: 0
container: "system"
package_map: "./tests/package.map"
flag_map: "./tests/flag.map"
flag_val: "./tests/rw.flag.val"
package_map: "./tests/tmp.ro.package.map"
flag_map: "./tests/tmp.ro.flag.map"
flag_val: "./tests/tmp.rw.flag.val"
timestamp: 12345
}
"#;
@ -268,7 +305,7 @@ files {
let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
assert_eq!(
format!("{:?}", error),
"Cannot mmap file ./tests/rw.flag.val as it is not read only"
"MapFileFail(fail to map non read only storage file ./tests/tmp.rw.flag.val)"
);
}
}

38
tools/aconfig/aconfig_storage_file/src/package_table.rs
View file

@ -17,8 +17,9 @@
//! package table module defines the package table file format and methods for serialization
//! and deserialization
use crate::AconfigStorageError::{self, BytesParseFail, HigherStorageFileVersion};
use crate::{get_bucket_index, read_str_from_bytes, read_u32_from_bytes};
use anyhow::{anyhow, Result};
use anyhow::anyhow;
/// Package table header struct
#[derive(PartialEq, Debug)]
@ -47,7 +48,7 @@ impl PackageTableHeader {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let mut head = 0;
Ok(Self {
version: read_u32_from_bytes(bytes, &mut head)?,
@ -85,7 +86,7 @@ impl PackageTableNode {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let mut head = 0;
let node = Self {
package_name: read_str_from_bytes(bytes, &mut head)?,
@ -127,7 +128,7 @@ impl PackageTable {
}
/// Deserialize from bytes
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AconfigStorageError> {
let header = PackageTableHeader::from_bytes(bytes)?;
let num_packages = header.num_packages;
let num_buckets = crate::get_table_size(num_packages)?;
@ -144,7 +145,8 @@ impl PackageTable {
head += node.as_bytes().len();
Ok(node)
})
.collect::<Result<Vec<_>>>()?;
.collect::<Result<Vec<_>, AconfigStorageError>>()
.map_err(|errmsg| BytesParseFail(anyhow!("fail to parse package table: {}", errmsg)))?;
let table = Self { header, buckets, nodes };
Ok(table)
@ -159,18 +161,21 @@ pub struct PackageOffset {
}
/// Query package id and start offset
pub fn find_package_offset(buf: &[u8], package: &str) -> Result<Option<PackageOffset>> {
pub fn find_package_offset(
buf: &[u8],
package: &str,
) -> Result<Option<PackageOffset>, AconfigStorageError> {
let interpreted_header = PackageTableHeader::from_bytes(buf)?;
if interpreted_header.version > crate::FILE_VERSION {
return Err(anyhow!(
return Err(HigherStorageFileVersion(anyhow!(
"Cannot read storage file with a higher version of {} with lib version {}",
interpreted_header.version,
crate::FILE_VERSION
));
)));
}
let num_buckets = (interpreted_header.node_offset - interpreted_header.bucket_offset) / 4;
let bucket_index = PackageTableNode::find_bucket_index(&package, num_buckets);
let bucket_index = PackageTableNode::find_bucket_index(package, num_buckets);
let mut pos = (interpreted_header.bucket_offset + 4 * bucket_index) as usize;
let mut package_node_offset = read_u32_from_bytes(buf, &mut pos)? as usize;
@ -199,7 +204,7 @@ pub fn find_package_offset(buf: &[u8], package: &str) -> Result<Option<PackageOf
mod tests {
use super::*;
pub fn create_test_package_table() -> Result<PackageTable> {
pub fn create_test_package_table() -> PackageTable {
let header = PackageTableHeader {
version: crate::FILE_VERSION,
container: String::from("system"),
@ -228,14 +233,13 @@ mod tests {
next_offset: None,
};
let nodes = vec![first_node, second_node, third_node];
Ok(PackageTable { header, buckets, nodes })
PackageTable { header, buckets, nodes }
}
#[test]
// this test point locks down the table serialization
fn test_serialization() {
let package_table = create_test_package_table().unwrap();
let package_table = create_test_package_table();
let header: &PackageTableHeader = &package_table.header;
let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes());
assert!(reinterpreted_header.is_ok());
@ -255,7 +259,7 @@ mod tests {
#[test]
// this test point locks down table query
fn test_package_query() {
let package_table = create_test_package_table().unwrap().as_bytes();
let package_table = create_test_package_table().as_bytes();
let package_offset =
find_package_offset(&package_table[..], "com.android.aconfig.storage.test_1")
.unwrap()
@ -280,7 +284,7 @@ mod tests {
// this test point locks down table query of a non exist package
fn test_not_existed_package_query() {
// this will land at an empty bucket
let package_table = create_test_package_table().unwrap().as_bytes();
let package_table = create_test_package_table().as_bytes();
let package_offset =
find_package_offset(&package_table[..], "com.android.aconfig.storage.test_3").unwrap();
assert_eq!(package_offset, None);
@ -293,7 +297,7 @@ mod tests {
#[test]
// this test point locks down query error when file has a higher version
fn test_higher_version_storage_file() {
let mut table = create_test_package_table().unwrap();
let mut table = create_test_package_table();
table.header.version = crate::FILE_VERSION + 1;
let package_table = table.as_bytes();
let error = find_package_offset(&package_table[..], "com.android.aconfig.storage.test_1")
@ -301,7 +305,7 @@ mod tests {
assert_eq!(
format!("{:?}", error),
format!(
"Cannot read storage file with a higher version of {} with lib version {}",
"HigherStorageFileVersion(Cannot read storage file with a higher version of {} with lib version {})",
crate::FILE_VERSION + 1,
crate::FILE_VERSION
)

60
tools/aconfig/aconfig_storage_file/src/test_utils.rs
View file

@ -17,18 +17,74 @@
use crate::protos::ProtoStorageFiles;
use anyhow::Result;
use protobuf::Message;
use std::fs;
use std::io::Write;
use std::path::Path;
use std::sync::Once;
use tempfile::NamedTempFile;
pub fn get_binary_storage_proto_bytes(text_proto: &str) -> Result<Vec<u8>> {
static INIT: Once = Once::new();
pub(crate) fn get_binary_storage_proto_bytes(text_proto: &str) -> Result<Vec<u8>> {
let storage_files: ProtoStorageFiles = protobuf::text_format::parse_from_str(text_proto)?;
let mut binary_proto = Vec::new();
storage_files.write_to_vec(&mut binary_proto)?;
Ok(binary_proto)
}
pub fn write_bytes_to_temp_file(bytes: &[u8]) -> Result<NamedTempFile> {
pub(crate) fn write_bytes_to_temp_file(bytes: &[u8]) -> Result<NamedTempFile> {
let mut file = NamedTempFile::new()?;
let _ = file.write_all(&bytes);
Ok(file)
}
fn has_same_content(file1: &Path, file2: &Path) -> Result<bool> {
let bytes1 = fs::read(file1)?;
let bytes2 = fs::read(file2)?;
if bytes1.len() != bytes2.len() {
return Ok(false);
}
for (i, &b1) in bytes1.iter().enumerate() {
if b1 != bytes2[i] {
return Ok(false);
}
}
Ok(true)
}
pub(crate) fn create_temp_storage_files_for_test() {
INIT.call_once(|| {
let file_paths = [
("./tests/package.map", "./tests/tmp.ro.package.map"),
("./tests/flag.map", "./tests/tmp.ro.flag.map"),
("./tests/flag.val", "./tests/tmp.ro.flag.val"),
("./tests/package.map", "./tests/tmp.rw.package.map"),
("./tests/flag.map", "./tests/tmp.rw.flag.map"),
("./tests/flag.val", "./tests/tmp.rw.flag.val"),
];
for (file_path, copied_file_path) in file_paths.into_iter() {
let file_path = Path::new(&file_path);
let copied_file_path = Path::new(&copied_file_path);
if copied_file_path.exists() && !has_same_content(file_path, copied_file_path).unwrap()
{
fs::remove_file(copied_file_path).unwrap();
}
if !copied_file_path.exists() {
fs::copy(file_path, copied_file_path).unwrap();
}
}
});
}
pub(crate) fn set_temp_storage_files_to_read_only() {
let file_paths =
["./tests/tmp.ro.package.map", "./tests/tmp.ro.flag.map", "./tests/tmp.ro.flag.val"];
for file_path in file_paths.into_iter() {
let file_path = Path::new(&file_path);
let mut perms = fs::metadata(file_path).unwrap().permissions();
if !perms.readonly() {
perms.set_readonly(true);
fs::set_permissions(file_path, perms).unwrap();
}
}
}

BIN
tools/aconfig/aconfig_storage_file/tests/rw.flag.map
View file

Binary file not shown.

BIN
tools/aconfig/aconfig_storage_file/tests/rw.flag.val
View file

Binary file not shown.

BIN
tools/aconfig/aconfig_storage_file/tests/rw.package.map
View file

Binary file not shown.