diff --git a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp index bf915603bc..4411b90097 100644 --- a/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp +++ b/neuralnetworks/1.2/vts/functional/CompilationCachingTests.cpp @@ -45,9 +45,9 @@ using ::android::hardware::neuralnetworks::V1_2::implementation::PreparedModelCa using ::android::nn::allocateSharedMemory; using ::test_helper::MixedTypedExample; -namespace { +namespace float32_model { -// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of mobilenet. +// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of float32 mobilenet. #include "examples/mobilenet_224_gender_basic_fixed.example.cpp" #include "vts_models/mobilenet_224_gender_basic_fixed.model.cpp" @@ -55,6 +55,44 @@ namespace { [[maybe_unused]] auto dummy_createTestModel = createTestModel_dynamic_output_shape; [[maybe_unused]] auto dummy_get_examples = get_examples_dynamic_output_shape; +// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h. +// This function assumes the operation is always ADD. +std::vector getLargeModelExamples(uint32_t len) { + float outputValue = 1.0f + static_cast(len); + return {{.operands = { + // Input + {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {1.0f}}}}, + // Output + {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {outputValue}}}}}}}; +} + +} // namespace float32_model + +namespace quant8_model { + +// In frameworks/ml/nn/runtime/test/generated/, creates a hidl model of quant8 mobilenet. +#include "examples/mobilenet_quantized.example.cpp" +#include "vts_models/mobilenet_quantized.model.cpp" + +// Prevent the compiler from complaining about an otherwise unused function. +[[maybe_unused]] auto dummy_createTestModel = createTestModel_dynamic_output_shape; +[[maybe_unused]] auto dummy_get_examples = get_examples_dynamic_output_shape; + +// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h. +// This function assumes the operation is always ADD. +std::vector getLargeModelExamples(uint32_t len) { + uint8_t outputValue = 1 + static_cast(len); + return {{.operands = {// Input + {.operandDimensions = {{0, {1}}}, .quant8AsymmOperands = {{0, {1}}}}, + // Output + {.operandDimensions = {{0, {1}}}, + .quant8AsymmOperands = {{0, {outputValue}}}}}}}; +} + +} // namespace quant8_model + +namespace { + enum class AccessMode { READ_WRITE, READ_ONLY, WRITE_ONLY }; // Creates cache handles based on provided file groups. @@ -101,14 +139,18 @@ void createCacheHandles(const std::vector>& fileGroups, // ↑ ↑ ↑ ↑ // [1] [1] [1] [1] // -Model createLargeTestModel(OperationType op, uint32_t len) { +// This function assumes the operation is either ADD or MUL. +template +Model createLargeTestModelImpl(OperationType op, uint32_t len) { + EXPECT_TRUE(op == OperationType::ADD || op == OperationType::MUL); + // Model operations and operands. std::vector operations(len); std::vector operands(len * 2 + 2); // The constant buffer pool. This contains the activation scalar, followed by the // per-operation constant operands. - std::vector operandValues(sizeof(int32_t) + len * sizeof(float)); + std::vector operandValues(sizeof(int32_t) + len * sizeof(CppType)); // The activation scalar, value = 0. operands[0] = { @@ -122,7 +164,26 @@ Model createLargeTestModel(OperationType op, uint32_t len) { }; memset(operandValues.data(), 0, sizeof(int32_t)); - const float floatBufferValue = 1.0f; + // The buffer value of the constant second operand. The logical value is always 1.0f. + CppType bufferValue; + // The scale of the first and second operand. + float scale1, scale2; + if (operandType == OperandType::TENSOR_FLOAT32) { + bufferValue = 1.0f; + scale1 = 0.0f; + scale2 = 0.0f; + } else if (op == OperationType::ADD) { + bufferValue = 1; + scale1 = 1.0f; + scale2 = 1.0f; + } else { + // To satisfy the constraint on quant8 MUL: input0.scale * input1.scale < output.scale, + // set input1 to have scale = 0.5f and bufferValue = 2, i.e. 1.0f in floating point. + bufferValue = 2; + scale1 = 1.0f; + scale2 = 0.5f; + } + for (uint32_t i = 0; i < len; i++) { const uint32_t firstInputIndex = i * 2 + 1; const uint32_t secondInputIndex = firstInputIndex + 1; @@ -130,10 +191,10 @@ Model createLargeTestModel(OperationType op, uint32_t len) { // The first operation input. operands[firstInputIndex] = { - .type = OperandType::TENSOR_FLOAT32, + .type = operandType, .dimensions = {1}, .numberOfConsumers = 1, - .scale = 0.0f, + .scale = scale1, .zeroPoint = 0, .lifetime = (i == 0 ? OperandLifeTime::MODEL_INPUT : OperandLifeTime::TEMPORARY_VARIABLE), @@ -142,18 +203,18 @@ Model createLargeTestModel(OperationType op, uint32_t len) { // The second operation input, value = 1. operands[secondInputIndex] = { - .type = OperandType::TENSOR_FLOAT32, + .type = operandType, .dimensions = {1}, .numberOfConsumers = 1, - .scale = 0.0f, + .scale = scale2, .zeroPoint = 0, .lifetime = OperandLifeTime::CONSTANT_COPY, .location = {.poolIndex = 0, - .offset = static_cast(i * sizeof(float) + sizeof(int32_t)), - .length = sizeof(float)}, + .offset = static_cast(i * sizeof(CppType) + sizeof(int32_t)), + .length = sizeof(CppType)}, }; - memcpy(operandValues.data() + sizeof(int32_t) + i * sizeof(float), &floatBufferValue, - sizeof(float)); + memcpy(operandValues.data() + sizeof(int32_t) + i * sizeof(CppType), &bufferValue, + sizeof(CppType)); // The operation. All operations share the same activation scalar. // The output operand is created as an input in the next iteration of the loop, in the case @@ -168,10 +229,10 @@ Model createLargeTestModel(OperationType op, uint32_t len) { // The model output. operands.back() = { - .type = OperandType::TENSOR_FLOAT32, + .type = operandType, .dimensions = {1}, .numberOfConsumers = 0, - .scale = 0.0f, + .scale = scale1, .zeroPoint = 0, .lifetime = OperandLifeTime::MODEL_OUTPUT, .location = {}, @@ -191,22 +252,13 @@ Model createLargeTestModel(OperationType op, uint32_t len) { }; } -// MixedTypedExample is defined in frameworks/ml/nn/tools/test_generator/include/TestHarness.h. -// This function assumes the operation is always ADD. -std::vector getLargeModelExamples(uint32_t len) { - float outputValue = 1.0f + static_cast(len); - return {{.operands = { - // Input - {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {1.0f}}}}, - // Output - {.operandDimensions = {{0, {1}}}, .float32Operands = {{0, {outputValue}}}}}}}; -}; - } // namespace // Tag for the compilation caching tests. -class CompilationCachingTest : public NeuralnetworksHidlTest { +class CompilationCachingTestBase : public NeuralnetworksHidlTest { protected: + CompilationCachingTestBase(OperandType type) : kOperandType(type) {} + void SetUp() override { NeuralnetworksHidlTest::SetUp(); ASSERT_NE(device.get(), nullptr); @@ -263,6 +315,40 @@ class CompilationCachingTest : public NeuralnetworksHidlTest { NeuralnetworksHidlTest::TearDown(); } + // Model and examples creators. According to kOperandType, the following methods will return + // either float32 model/examples or the quant8 variant. + Model createTestModel() { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return float32_model::createTestModel(); + } else { + return quant8_model::createTestModel(); + } + } + + std::vector get_examples() { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return float32_model::get_examples(); + } else { + return quant8_model::get_examples(); + } + } + + Model createLargeTestModel(OperationType op, uint32_t len) { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return createLargeTestModelImpl(op, len); + } else { + return createLargeTestModelImpl(op, len); + } + } + + std::vector getLargeModelExamples(uint32_t len) { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return float32_model::getLargeModelExamples(len); + } else { + return quant8_model::getLargeModelExamples(len); + } + } + // See if the service can handle the model. bool isModelFullySupported(const V1_2::Model& model) { bool fullySupportsModel = false; @@ -366,9 +452,20 @@ class CompilationCachingTest : public NeuralnetworksHidlTest { uint32_t mNumModelCache; uint32_t mNumDataCache; uint32_t mIsCachingSupported; + + // The primary data type of the testModel. + const OperandType kOperandType; }; -TEST_F(CompilationCachingTest, CacheSavingAndRetrieval) { +// A parameterized fixture of CompilationCachingTestBase. Every test will run twice, with the first +// pass running with float32 models and the second pass running with quant8 models. +class CompilationCachingTest : public CompilationCachingTestBase, + public ::testing::WithParamInterface { + protected: + CompilationCachingTest() : CompilationCachingTestBase(GetParam()) {} +}; + +TEST_P(CompilationCachingTest, CacheSavingAndRetrieval) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -409,7 +506,7 @@ TEST_F(CompilationCachingTest, CacheSavingAndRetrieval) { /*testDynamicOutputShape=*/false); } -TEST_F(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { +TEST_P(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -472,7 +569,7 @@ TEST_F(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { /*testDynamicOutputShape=*/false); } -TEST_F(CompilationCachingTest, SaveToCacheInvalidNumCache) { +TEST_P(CompilationCachingTest, SaveToCacheInvalidNumCache) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -584,7 +681,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidNumCache) { } } -TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { +TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -664,7 +761,7 @@ TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { } } -TEST_F(CompilationCachingTest, SaveToCacheInvalidNumFd) { +TEST_P(CompilationCachingTest, SaveToCacheInvalidNumFd) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -776,7 +873,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidNumFd) { } } -TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) { +TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -856,7 +953,7 @@ TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidNumFd) { } } -TEST_F(CompilationCachingTest, SaveToCacheInvalidAccessMode) { +TEST_P(CompilationCachingTest, SaveToCacheInvalidAccessMode) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -914,7 +1011,7 @@ TEST_F(CompilationCachingTest, SaveToCacheInvalidAccessMode) { } } -TEST_F(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) { +TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) { // Create test HIDL model and compile. const Model testModel = createTestModel(); if (checkEarlyTermination(testModel)) return; @@ -990,7 +1087,7 @@ static void copyCacheFiles(const std::vector>& from, constexpr uint32_t kLargeModelSize = 100; constexpr uint32_t kNumIterationsTOCTOU = 100; -TEST_F(CompilationCachingTest, SaveToCache_TOCTOU) { +TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { if (!mIsCachingSupported) return; // Create test models and check if fully supported by the service. @@ -1053,7 +1150,7 @@ TEST_F(CompilationCachingTest, SaveToCache_TOCTOU) { } } -TEST_F(CompilationCachingTest, PrepareFromCache_TOCTOU) { +TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) { if (!mIsCachingSupported) return; // Create test models and check if fully supported by the service. @@ -1116,7 +1213,7 @@ TEST_F(CompilationCachingTest, PrepareFromCache_TOCTOU) { } } -TEST_F(CompilationCachingTest, ReplaceSecuritySensitiveCache) { +TEST_P(CompilationCachingTest, ReplaceSecuritySensitiveCache) { if (!mIsCachingSupported) return; // Create test models and check if fully supported by the service. @@ -1164,11 +1261,19 @@ TEST_F(CompilationCachingTest, ReplaceSecuritySensitiveCache) { } } -class CompilationCachingSecurityTest : public CompilationCachingTest, - public ::testing::WithParamInterface { +static const auto kOperandTypeChoices = + ::testing::Values(OperandType::TENSOR_FLOAT32, OperandType::TENSOR_QUANT8_ASYMM); + +INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingTest, kOperandTypeChoices); + +class CompilationCachingSecurityTest + : public CompilationCachingTestBase, + public ::testing::WithParamInterface> { protected: + CompilationCachingSecurityTest() : CompilationCachingTestBase(std::get<0>(GetParam())) {} + void SetUp() { - CompilationCachingTest::SetUp(); + CompilationCachingTestBase::SetUp(); generator.seed(kSeed); } @@ -1254,7 +1359,7 @@ class CompilationCachingSecurityTest : public CompilationCachingTest, } } - const uint32_t kSeed = GetParam(); + const uint32_t kSeed = std::get<1>(GetParam()); std::mt19937 generator; }; @@ -1302,7 +1407,7 @@ TEST_P(CompilationCachingSecurityTest, WrongToken) { } INSTANTIATE_TEST_CASE_P(TestCompilationCaching, CompilationCachingSecurityTest, - ::testing::Range(0U, 10U)); + ::testing::Combine(kOperandTypeChoices, ::testing::Range(0U, 10U))); } // namespace functional } // namespace vts