Merge "Create EmuMetadataGenerator to check meta.json." into main

This commit is contained in:
Yu Shan 2024-02-07 23:10:54 +00:00 committed by Gerrit Code Review
commit 1652bf307c
8 changed files with 2928 additions and 4738 deletions

View file

@ -0,0 +1,26 @@
/*
* Copyright (C) 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
filegroup {
name: "android.hardware.automotive.vehicle-types-meta",
srcs: [
"android.hardware.automotive.vehicle-types-meta.json",
],
}

View file

@ -1,136 +0,0 @@
#!/usr/bin/python3
#
# Script for generation of VHAL properties metadata .json from AIDL interface
#
# This metadata is used to display human property names, names of enum
# data types for their values, change and access modes and other information,
# available from AIDL block comments, but not at runtime.
#
# Usage example:
# ./emu_metadata/generate_emulator_metadata.py android/hardware/automotive/vehicle $OUT/android.hardware.automotive.vehicle-types-meta.json
# (Note, that the resulting file has to match a '*types-meta.json' pattern to be parsed by the emulator).
#
import json
import os
import re
import sys
from pathlib import Path
RE_PACKAGE = re.compile(r"\npackage\s([\.a-z0-9]*);")
RE_IMPORT = re.compile(r"\nimport\s([\.a-zA-Z0-9]*);")
RE_ENUM = re.compile(r"\s*enum\s+(\w*) {\n(.*)}", re.MULTILINE | re.DOTALL)
RE_COMMENT = re.compile(r"(?:(?:\/\*\*)((?:.|\n)*?)(?:\*\/))?(?:\n|^)\s*(\w*)(?:\s+=\s*)?((?:[\.\-a-zA-Z0-9]|\s|\+|)*),",
re.DOTALL)
RE_BLOCK_COMMENT_TITLE = re.compile("^(?:\s|\*)*((?:\w|\s|\.)*)\n(?:\s|\*)*(?:\n|$)")
RE_BLOCK_COMMENT_ANNOTATION = re.compile("^(?:\s|\*)*@(\w*)\s+((?:[\w:\.])*)", re.MULTILINE)
RE_HEX_NUMBER = re.compile("([\.\-0-9A-Za-z]+)")
class JEnum:
def __init__(self, package, name):
self.package = package
self.name = name
self.values = []
class Enum:
def __init__(self, package, name, text, imports):
self.text = text
self.parsed = False
self.imports = imports
self.jenum = JEnum(package, name)
def parse(self, enums):
if self.parsed:
return
for dep in self.imports:
enums[dep].parse(enums)
print("Parsing " + self.jenum.name)
matches = RE_COMMENT.findall(self.text)
defaultValue = 0
for match in matches:
value = dict()
value['name'] = match[1]
value['value'] = self.calculateValue(match[2], defaultValue, enums)
defaultValue = value['value'] + 1
if self.jenum.name == "VehicleProperty":
block_comment = match[0]
self.parseBlockComment(value, block_comment)
self.jenum.values.append(value)
self.parsed = True
self.text = None
def get_value(self, value_name):
for value in self.jenum.values:
if value['name'] == value_name:
return value['value']
raise Exception("Cannot decode value: " + self.jenum.package + " : " + value_name)
def calculateValue(self, expression, default_value, enums):
numbers = RE_HEX_NUMBER.findall(expression)
if len(numbers) == 0:
return default_value
result = 0
base = 10
if numbers[0].lower().startswith("0x"):
base = 16
for number in numbers:
if '.' in number:
package, val_name = number.split('.')
for dep in self.imports:
if package in dep:
result += enums[dep].get_value(val_name)
else:
result += int(number, base)
return result
def parseBlockComment(self, value, blockComment):
titles = RE_BLOCK_COMMENT_TITLE.findall(blockComment)
for title in titles:
value['name'] = title
break
annots_res = RE_BLOCK_COMMENT_ANNOTATION.findall(blockComment)
for annot in annots_res:
value[annot[0]] = annot[1].replace(".", ":")
class Converter:
# Only addition is supported for now, but that covers all existing properties except
# OBD diagnostics, which use bitwise shifts
def convert(self, input):
text = Path(input).read_text()
matches = RE_ENUM.findall(text)
package = RE_PACKAGE.findall(text)[0]
imports = RE_IMPORT.findall(text)
enums = []
for match in matches:
enum = Enum(package, match[0], match[1], imports)
enums.append(enum)
return enums
def main():
if (len(sys.argv) != 3):
print("Usage: ", sys.argv[0], " INPUT_PATH OUTPUT")
sys.exit(1)
aidl_path = sys.argv[1]
out_path = sys.argv[2]
enums_dict = dict()
for file in os.listdir(aidl_path):
enums = Converter().convert(os.path.join(aidl_path, file))
for enum in enums:
enums_dict[enum.jenum.package + "." + enum.jenum.name] = enum
result = []
for enum_name, enum in enums_dict.items():
enum.parse(enums_dict)
result.append(enum.jenum.__dict__)
json_result = json.dumps(result, default=None, indent=2)
with open(out_path, 'w') as f:
f.write(json_result)
if __name__ == "__main__":
main()

View file

@ -55,6 +55,10 @@ cc_library {
"src/ConnectedClient.cpp",
"src/DefaultVehicleHal.cpp",
"src/SubscriptionManager.cpp",
// A target to check whether the file
// android.hardware.automotive.vehicle-types-meta.json needs update.
// The output is just an empty cpp file and not actually used.
":check_generated_enum_metadata_json",
],
static_libs: [
"VehicleHalUtils",

View file

@ -57,5 +57,11 @@ aidl_interface {
},
],
}
filegroup {
name: "android.hardware.automotive.vehicle.property-files",
srcs: [
"android/hardware/automotive/vehicle/*.aidl",
],
}

View file

@ -0,0 +1,60 @@
/*
* Copyright (C) 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
java_binary_host {
name: "EnumMetadataGenerator",
srcs: ["src/**/*.java"],
manifest: "manifest.txt",
static_libs: [
"javaparser",
"javaparser-symbol-solver",
"json-prebuilt",
"androidx.annotation_annotation",
],
}
// A rule to convert VHAL property AIDL files to java files.
gensrcs {
name: "gen_vehicle_property_java_file",
srcs: [
":android.hardware.automotive.vehicle.property-files",
],
tools: ["aidl"],
cmd: "$(location aidl) --lang=java --structured --stability=vintf $(in) -I hardware/interfaces/automotive/vehicle/aidl_property --out $(genDir)/hardware/interfaces/automotive/vehicle/aidl_property",
output_extension: "java",
}
// A target to check whether android.hardware.automotive.vehicle-types-meta.json
// needs to be updated. The output is just an empty cpp file to be included
// in the higher-level build target.
// It will generate generated.json at output directory based on VHAL property
// java files and check it against
// android.hardware.automotive.vehicle-types-meta.json. If not the same, the
// build will fail.
genrule {
name: "check_generated_enum_metadata_json",
tools: ["EnumMetadataGenerator"],
srcs: [
":android.hardware.automotive.vehicle-types-meta",
":gen_vehicle_property_java_file",
],
cmd: "$(location EnumMetadataGenerator) --check_against $(location :android.hardware.automotive.vehicle-types-meta) --output_empty_file $(out) --output_json $(genDir)/generate_enum_metadata.json --input_files $(locations :gen_vehicle_property_java_file)",
out: ["generate_enum_metadata_checked.cpp"],
}

View file

@ -0,0 +1 @@
Main-Class: com.android.car.tool.EmuMetadataGenerator

View file

@ -0,0 +1,403 @@
/*
* Copyright (C) 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.car.tool;
import com.github.javaparser.StaticJavaParser;
import com.github.javaparser.ast.CompilationUnit;
import com.github.javaparser.ast.body.AnnotationDeclaration;
import com.github.javaparser.ast.body.FieldDeclaration;
import com.github.javaparser.ast.body.VariableDeclarator;
import com.github.javaparser.ast.comments.Comment;
import com.github.javaparser.ast.expr.AnnotationExpr;
import com.github.javaparser.ast.expr.ArrayInitializerExpr;
import com.github.javaparser.ast.expr.Expression;
import com.github.javaparser.ast.expr.NormalAnnotationExpr;
import com.github.javaparser.ast.expr.SingleMemberAnnotationExpr;
import com.github.javaparser.ast.expr.UnaryExpr;
import com.github.javaparser.ast.type.ClassOrInterfaceType;
import com.github.javaparser.javadoc.Javadoc;
import com.github.javaparser.javadoc.JavadocBlockTag;
import com.github.javaparser.javadoc.description.JavadocDescription;
import com.github.javaparser.javadoc.description.JavadocDescriptionElement;
import com.github.javaparser.javadoc.description.JavadocInlineTag;
import com.github.javaparser.resolution.declarations.ResolvedFieldDeclaration;
import com.github.javaparser.resolution.declarations.ResolvedReferenceTypeDeclaration;
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
import com.github.javaparser.symbolsolver.javaparsermodel.declarations.JavaParserFieldDeclaration;
import com.github.javaparser.symbolsolver.model.resolution.TypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.json.JSONArray;
import org.json.JSONObject;
public final class EmuMetadataGenerator {
private static final String DEFAULT_PACKAGE_NAME = "android.hardware.automotive.vehicle";
private static final String INPUT_DIR_OPTION = "--input_dir";
private static final String INPUT_FILES_OPTION = "--input_files";
private static final String PACKAGE_NAME_OPTION = "--package_name";
private static final String OUTPUT_JSON_OPTION = "--output_json";
private static final String OUTPUT_EMPTY_FILE_OPTION = "--output_empty_file";
private static final String CHECK_AGAINST_OPTION = "--check_against";
private static final String USAGE = "EnumMetadataGenerator " + INPUT_DIR_OPTION
+ " [path_to_aidl_gen_dir] " + INPUT_FILES_OPTION + " [input_files] "
+ PACKAGE_NAME_OPTION + " [package_name] " + OUTPUT_JSON_OPTION + " [output_json] "
+ OUTPUT_EMPTY_FILE_OPTION + " [output_header_file] " + CHECK_AGAINST_OPTION
+ " [json_file_to_check_against]\n"
+ "Parses the VHAL property AIDL interface generated Java files to a json file to be"
+ " used by emulator\n"
+ "Options: \n" + INPUT_DIR_OPTION
+ ": the path to a directory containing AIDL interface Java files, "
+ "either this or input_files must be specified\n" + INPUT_FILES_OPTION
+ ": one or more Java files, this is used to decide the input "
+ "directory\n" + PACKAGE_NAME_OPTION
+ ": the optional package name for the interface, by default is " + DEFAULT_PACKAGE_NAME
+ "\n" + OUTPUT_JSON_OPTION + ": The output JSON file\n" + OUTPUT_EMPTY_FILE_OPTION
+ ": Only used for check_mode, this file will be created if "
+ "check passed\n" + CHECK_AGAINST_OPTION
+ ": An optional JSON file to check against. If specified, the "
+ "generated output file will be checked against this file, if they are not the same, "
+ "the script will fail, otherwise, the output_empty_file will be created\n"
+ "For example: \n"
+ "EnumMetadataGenerator --input_dir out/soong/.intermediates/hardware/"
+ "interfaces/automotive/vehicle/aidl_property/android.hardware.automotive.vehicle."
+ "property-V3-java-source/gen/ --package_name android.hardware.automotive.vehicle "
+ "--output_json /tmp/android.hardware.automotive.vehicle-types-meta.json";
private static final String VEHICLE_PROPERTY_FILE = "VehicleProperty.java";
private static final String CHECK_FILE_PATH =
"${ANDROID_BUILD_TOP}/hardware/interfaces/automotive/vehicle/aidl/emu_metadata/"
+ "android.hardware.automotive.vehicle-types-meta.json";
// Emulator can display at least this many characters before cutting characters.
private static final int MAX_PROPERTY_NAME_LENGTH = 30;
/**
* Parses the enum field declaration as an int value.
*/
private static int parseIntEnumField(FieldDeclaration fieldDecl) {
VariableDeclarator valueDecl = fieldDecl.getVariables().get(0);
Expression expr = valueDecl.getInitializer().get();
if (expr.isIntegerLiteralExpr()) {
return expr.asIntegerLiteralExpr().asInt();
}
// For case like -123
if (expr.isUnaryExpr() && expr.asUnaryExpr().getOperator() == UnaryExpr.Operator.MINUS) {
return -expr.asUnaryExpr().getExpression().asIntegerLiteralExpr().asInt();
}
System.out.println("Unsupported expression: " + expr);
System.exit(1);
return 0;
}
private static boolean isPublicAndStatic(FieldDeclaration fieldDecl) {
return fieldDecl.isPublic() && fieldDecl.isStatic();
}
private static String getFieldName(FieldDeclaration fieldDecl) {
VariableDeclarator valueDecl = fieldDecl.getVariables().get(0);
return valueDecl.getName().asString();
}
private static class Enum {
Enum(String name, String packageName) {
this.name = name;
this.packageName = packageName;
}
public String name;
public String packageName;
public final List<ValueField> valueFields = new ArrayList<>();
}
private static class ValueField {
public String name;
public Integer value;
public final List<String> dataEnums = new ArrayList<>();
ValueField(String name, Integer value) {
this.name = name;
this.value = value;
}
}
private static Enum parseEnumInterface(
String inputDir, String dirName, String packageName, String enumName) throws Exception {
Enum enumIntf = new Enum(enumName, packageName);
CompilationUnit cu = StaticJavaParser.parse(new File(
inputDir + File.separator + dirName + File.separator + enumName + ".java"));
AnnotationDeclaration vehiclePropertyIdsClass =
cu.getAnnotationDeclarationByName(enumName).get();
List<FieldDeclaration> variables = vehiclePropertyIdsClass.findAll(FieldDeclaration.class);
for (int i = 0; i < variables.size(); i++) {
FieldDeclaration propertyDef = variables.get(i).asFieldDeclaration();
if (!isPublicAndStatic(propertyDef)) {
continue;
}
ValueField field =
new ValueField(getFieldName(propertyDef), parseIntEnumField(propertyDef));
enumIntf.valueFields.add(field);
}
return enumIntf;
}
// A hacky way to make the key in-order in the JSON object.
private static final class OrderedJSONObject extends JSONObject {
OrderedJSONObject() {
try {
Field map = JSONObject.class.getDeclaredField("nameValuePairs");
map.setAccessible(true);
map.set(this, new LinkedHashMap<>());
map.setAccessible(false);
} catch (IllegalAccessException | NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
}
private static String readFileContent(String fileName) throws Exception {
StringBuffer contentBuffer = new StringBuffer();
int bufferSize = 1024;
try (BufferedReader reader = new BufferedReader(new FileReader(fileName))) {
char buffer[] = new char[bufferSize];
while (true) {
int read = reader.read(buffer, 0, bufferSize);
if (read == -1) {
break;
}
contentBuffer.append(buffer, 0, read);
}
}
return contentBuffer.toString();
}
private static final class Args {
public final String inputDir;
public final String pkgName;
public final String pkgDir;
public final String output;
public final String checkFile;
public final String outputEmptyFile;
public Args(String[] args) throws IllegalArgumentException {
Map<String, List<String>> valuesByKey = new LinkedHashMap<>();
String key = null;
for (int i = 0; i < args.length; i++) {
String arg = args[i];
if (arg.startsWith("--")) {
key = arg;
continue;
}
if (key == null) {
throw new IllegalArgumentException("Missing key for value: " + arg);
}
if (valuesByKey.get(key) == null) {
valuesByKey.put(key, new ArrayList<>());
}
valuesByKey.get(key).add(arg);
}
String pkgName;
List<String> values = valuesByKey.get(PACKAGE_NAME_OPTION);
if (values == null) {
pkgName = DEFAULT_PACKAGE_NAME;
} else {
pkgName = values.get(0);
}
String pkgDir = pkgName.replace(".", File.separator);
this.pkgName = pkgName;
this.pkgDir = pkgDir;
String inputDir;
values = valuesByKey.get(INPUT_DIR_OPTION);
if (values == null) {
List<String> inputFiles = valuesByKey.get(INPUT_FILES_OPTION);
if (inputFiles == null) {
throw new IllegalArgumentException("Either " + INPUT_DIR_OPTION + " or "
+ INPUT_FILES_OPTION + " must be specified");
}
inputDir = new File(inputFiles.get(0)).getParent().replace(pkgDir, "");
} else {
inputDir = values.get(0);
}
this.inputDir = inputDir;
values = valuesByKey.get(OUTPUT_JSON_OPTION);
if (values == null) {
throw new IllegalArgumentException(OUTPUT_JSON_OPTION + " must be specified");
}
this.output = values.get(0);
values = valuesByKey.get(CHECK_AGAINST_OPTION);
if (values != null) {
this.checkFile = values.get(0);
} else {
this.checkFile = null;
}
values = valuesByKey.get(OUTPUT_EMPTY_FILE_OPTION);
if (values != null) {
this.outputEmptyFile = values.get(0);
} else {
this.outputEmptyFile = null;
}
}
}
/**
* Main function.
*/
public static void main(final String[] args) throws Exception {
Args parsedArgs;
try {
parsedArgs = new Args(args);
} catch (IllegalArgumentException e) {
System.out.println("Invalid arguments: " + e.getMessage());
System.out.println(USAGE);
System.exit(1);
// Never reach here.
return;
}
TypeSolver typeSolver = new CombinedTypeSolver(
new ReflectionTypeSolver(), new JavaParserTypeSolver(parsedArgs.inputDir));
StaticJavaParser.getConfiguration().setSymbolResolver(new JavaSymbolSolver(typeSolver));
Enum vehicleProperty = new Enum("VehicleProperty", parsedArgs.pkgName);
CompilationUnit cu = StaticJavaParser.parse(new File(parsedArgs.inputDir + File.separator
+ parsedArgs.pkgDir + File.separator + VEHICLE_PROPERTY_FILE));
AnnotationDeclaration vehiclePropertyIdsClass =
cu.getAnnotationDeclarationByName("VehicleProperty").get();
Set<String> dataEnumTypes = new HashSet<>();
List<FieldDeclaration> variables = vehiclePropertyIdsClass.findAll(FieldDeclaration.class);
for (int i = 0; i < variables.size(); i++) {
FieldDeclaration propertyDef = variables.get(i).asFieldDeclaration();
if (!isPublicAndStatic(propertyDef)) {
continue;
}
String propertyName = getFieldName(propertyDef);
if (propertyName.equals("INVALID")) {
continue;
}
Optional<Comment> maybeComment = propertyDef.getComment();
if (!maybeComment.isPresent()) {
System.out.println("missing comment for property: " + propertyName);
System.exit(1);
}
Javadoc doc = maybeComment.get().asJavadocComment().parse();
int propertyId = parseIntEnumField(propertyDef);
// We use the first paragraph as the property's name
String propertyDescription = doc.getDescription().toText().split("\n\n")[0];
String name = propertyDescription;
if (propertyDescription.indexOf("\n") != -1
|| propertyDescription.length() > MAX_PROPERTY_NAME_LENGTH) {
// The description is too long, we just use the property name.
name = propertyName;
}
ValueField field = new ValueField(name, propertyId);
List<JavadocBlockTag> blockTags = doc.getBlockTags();
List<Integer> dataEnums = new ArrayList<>();
for (int j = 0; j < blockTags.size(); j++) {
String commentTagName = blockTags.get(j).getTagName();
String commentTagContent = blockTags.get(j).getContent().toText();
if (!commentTagName.equals("data_enum")) {
continue;
}
field.dataEnums.add(commentTagContent);
dataEnumTypes.add(commentTagContent);
}
vehicleProperty.valueFields.add(field);
}
List<Enum> enumTypes = new ArrayList<>();
enumTypes.add(vehicleProperty);
for (String dataEnumType : dataEnumTypes) {
Enum dataEnum = parseEnumInterface(
parsedArgs.inputDir, parsedArgs.pkgDir, parsedArgs.pkgName, dataEnumType);
enumTypes.add(dataEnum);
}
// Output enumTypes as JSON to output.
JSONArray jsonEnums = new JSONArray();
for (int i = 0; i < enumTypes.size(); i++) {
Enum enumType = enumTypes.get(i);
JSONObject jsonEnum = new OrderedJSONObject();
jsonEnum.put("name", enumType.name);
jsonEnum.put("package", enumType.packageName);
JSONArray values = new JSONArray();
jsonEnum.put("values", values);
for (int j = 0; j < enumType.valueFields.size(); j++) {
ValueField valueField = enumType.valueFields.get(j);
JSONObject jsonValueField = new OrderedJSONObject();
jsonValueField.put("name", valueField.name);
jsonValueField.put("value", valueField.value);
if (!valueField.dataEnums.isEmpty()) {
JSONArray jsonDataEnums = new JSONArray();
for (String dataEnum : valueField.dataEnums) {
jsonDataEnums.put(dataEnum);
}
jsonValueField.put("data_enums", jsonDataEnums);
// To be backward compatible with older format where data_enum is a single
// entry.
jsonValueField.put("data_enum", valueField.dataEnums.get(0));
}
values.put(jsonValueField);
}
jsonEnums.put(jsonEnum);
}
try (FileOutputStream outputStream = new FileOutputStream(parsedArgs.output)) {
outputStream.write(jsonEnums.toString(4).getBytes());
}
System.out.println("Input at folder: " + parsedArgs.inputDir
+ " successfully parsed. Output at: " + parsedArgs.output);
if (parsedArgs.checkFile != null) {
String checkFileContent = readFileContent(parsedArgs.checkFile);
String generatedFileContent = readFileContent(parsedArgs.output);
String generatedFilePath = new File(parsedArgs.output).getAbsolutePath();
if (!checkFileContent.equals(generatedFileContent)) {
System.out.println("The file: " + CHECK_FILE_PATH + " needs to be updated, run: "
+ "\n\ncp " + generatedFilePath + " " + CHECK_FILE_PATH + "\n");
System.exit(1);
}
if (parsedArgs.outputEmptyFile != null) {
try (FileOutputStream outputStream =
new FileOutputStream(parsedArgs.outputEmptyFile)) {
// Do nothing, just create the file.
}
}
}
}
}