From 5b0c58d1fb1297d2f404184f22fc773a09406d78 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Wed, 29 Jan 2020 13:24:33 -0800 Subject: [PATCH 1/5] Add memory domain VTS validation tests. Bug: 147777318 Test: 1.3 VTS with sample driver Change-Id: Ia2097345924726d8fb627845fd7438cc3eb35eb6 Merged-In: Ia2097345924726d8fb627845fd7438cc3eb35eb6 (cherry picked from commit fed2f5213f1e968d31b433cde00cba6b6bbeebc6) --- neuralnetworks/1.3/vts/functional/Android.bp | 1 + .../vts/functional/GeneratedTestHarness.cpp | 2 - .../1.3/vts/functional/MemoryDomainTests.cpp | 1166 +++++++++++++++++ .../vts/functional/VtsHalNeuralnetworks.cpp | 15 + .../1.3/vts/functional/VtsHalNeuralnetworks.h | 4 + 5 files changed, 1186 insertions(+), 2 deletions(-) create mode 100644 neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp diff --git a/neuralnetworks/1.3/vts/functional/Android.bp b/neuralnetworks/1.3/vts/functional/Android.bp index f9362676ec..545a5be74e 100644 --- a/neuralnetworks/1.3/vts/functional/Android.bp +++ b/neuralnetworks/1.3/vts/functional/Android.bp @@ -40,6 +40,7 @@ cc_test { "BasicTests.cpp", "CompilationCachingTests.cpp", "GeneratedTestHarness.cpp", + "MemoryDomainTests.cpp", "QualityOfServiceTests.cpp", "TestAssertions.cpp", "ValidateBurst.cpp", diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 5689a39e1b..ff21960e6a 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -72,8 +72,6 @@ using HidlToken = hidl_array(Constant::BYTE_SIZE_ namespace { -enum class Executor { ASYNC, SYNC, BURST, FENCED }; - enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE }; enum class MemoryType { SHARED, DEVICE }; diff --git a/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp b/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp new file mode 100644 index 0000000000..08c1b35510 --- /dev/null +++ b/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp @@ -0,0 +1,1166 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_hidl_hal_test" + +#include +#include + +#include "1.3/Callbacks.h" +#include "1.3/Utils.h" +#include "GeneratedTestHarness.h" +#include "MemoryUtils.h" +#include "TestHarness.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace android::hardware::neuralnetworks::V1_3::vts::functional { + +using namespace test_helper; +using implementation::ExecutionCallback; +using implementation::PreparedModelCallback; +using V1_0::RequestArgument; +using V1_1::ExecutionPreference; +using V1_2::Constant; +using V1_2::MeasureTiming; +using V1_2::OutputShape; +using V1_2::Timing; + +namespace { + +const auto kNamedDeviceChoices = testing::ValuesIn(getNamedDevices()); + +// A 1.3 driver is likely to support at least one of the following operand types. +const std::vector kTestOperandTypeChoicesVector = { + TestOperandType::TENSOR_FLOAT32, + TestOperandType::TENSOR_FLOAT16, + TestOperandType::TENSOR_QUANT8_ASYMM, + TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, +}; +const auto kTestOperandTypeChoices = testing::ValuesIn(kTestOperandTypeChoicesVector); + +bool isInChoices(TestOperandType type) { + return std::count(kTestOperandTypeChoicesVector.begin(), kTestOperandTypeChoicesVector.end(), + type) > 0; +} + +bool isFloat(TestOperandType type) { + CHECK(isInChoices(type)); + return type == TestOperandType::TENSOR_FLOAT32 || type == TestOperandType::TENSOR_FLOAT16; +} + +// Create dummy buffers for model constants as well as inputs and outputs. +// We only care about the size here because we will not check accuracy in validation tests. +void createDummyData(TestModel* testModel) { + for (auto& operand : testModel->main.operands) { + if (operand.data != nullptr) continue; + switch (operand.lifetime) { + case TestOperandLifeTime::SUBGRAPH_INPUT: + case TestOperandLifeTime::SUBGRAPH_OUTPUT: + case TestOperandLifeTime::CONSTANT_COPY: + case TestOperandLifeTime::CONSTANT_REFERENCE: { + const uint32_t size = nn::nonExtensionOperandSizeOfData( + static_cast(operand.type), operand.dimensions); + operand.data = TestBuffer(size); + } break; + default: + break; + } + } +} + +TestOperand createInt32Scalar(int32_t value) { + return { + .type = TestOperandType::INT32, + .dimensions = {}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY, + .data = TestBuffer::createFromVector({value}), + }; +} + +// Construct a test model with multiple CONV_2D operations with the given operand as inputs. +// The dimensions of the filters are chosen to ensure outputs has the same dimensions as inputs. +// We choose CONV_2D operation because it is commonly supported by most drivers. +TestModel createConvModel(const TestOperand& operand, uint32_t numOperations) { + CHECK(isInChoices(operand.type)); + + TestOperand weight = {.type = operand.type, + .dimensions = {operand.dimensions[3], 3, 3, operand.dimensions[3]}, + .numberOfConsumers = 1, + .scale = isFloat(operand.type) ? 0.0f : 1.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY}; + + TestOperand bias = { + .type = isFloat(operand.type) ? operand.type : TestOperandType::TENSOR_INT32, + .dimensions = {operand.dimensions[3]}, + .numberOfConsumers = 1, + .scale = operand.scale * weight.scale, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY}; + + TestOperand output = operand; + output.numberOfConsumers = 0; + output.lifetime = TestOperandLifeTime::SUBGRAPH_OUTPUT; + + const std::vector operands = { + operand, + std::move(weight), + std::move(bias), + createInt32Scalar(1), // same padding + createInt32Scalar(1), // width stride + createInt32Scalar(1), // height stride + createInt32Scalar(0), // activation = NONE + std::move(output), + }; + + TestModel model; + for (uint32_t i = 0; i < numOperations; i++) { + model.main.operands.insert(model.main.operands.end(), operands.begin(), operands.end()); + const uint32_t inputIndex = operands.size() * i; + const uint32_t outputIndex = inputIndex + operands.size() - 1; + std::vector inputs(operands.size() - 1); + std::iota(inputs.begin(), inputs.end(), inputIndex); + model.main.operations.push_back({.type = TestOperationType::CONV_2D, + .inputs = std::move(inputs), + .outputs = {outputIndex}}); + model.main.inputIndexes.push_back(inputIndex); + model.main.outputIndexes.push_back(outputIndex); + } + createDummyData(&model); + return model; +} + +// Construct a test model with a single ADD operation with the given operand as input0 and input1. +// This is to cover additional cases that the CONV_2D model does not support, e.g. arbitrary input +// operand rank, scalar input operand. We choose ADD operation because it is commonly supported by +// most drivers. +TestModel createSingleAddModel(const TestOperand& operand) { + CHECK(isInChoices(operand.type)); + + TestOperand act = { + .type = TestOperandType::INT32, + .dimensions = {}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }; + + TestOperand output = operand; + output.numberOfConsumers = 0; + output.lifetime = TestOperandLifeTime::SUBGRAPH_OUTPUT; + + TestModel model = { + .main = + { + .operands = + { + operand, + operand, + std::move(act), + output, + }, + .operations = {{.type = TestOperationType::ADD, + .inputs = {0, 1, 2}, + .outputs = {3}}}, + .inputIndexes = {0, 1, 2}, + .outputIndexes = {3}, + }, + }; + createDummyData(&model); + return model; +} + +// A dummy invalid IPreparedModel class for MemoryDomainAllocateTest.InvalidPreparedModel +class InvalidPreparedModel : public IPreparedModel { + public: + Return execute(const V1_0::Request&, + const sp&) override { + return V1_0::ErrorStatus::GENERAL_FAILURE; + } + Return execute_1_2(const V1_0::Request&, V1_2::MeasureTiming, + const sp&) override { + return V1_0::ErrorStatus::GENERAL_FAILURE; + } + Return execute_1_3(const V1_3::Request&, V1_2::MeasureTiming, + const V1_3::OptionalTimePoint&, + const V1_3::OptionalTimeoutDuration&, + const sp&) override { + return V1_3::ErrorStatus::GENERAL_FAILURE; + } + Return executeSynchronously(const V1_0::Request&, V1_2::MeasureTiming, + executeSynchronously_cb) override { + return Void(); + } + Return executeSynchronously_1_3(const V1_3::Request&, V1_2::MeasureTiming, + const V1_3::OptionalTimePoint&, + const V1_3::OptionalTimeoutDuration&, + executeSynchronously_1_3_cb) override { + return Void(); + } + Return configureExecutionBurst(const sp&, + const MQDescriptorSync&, + const MQDescriptorSync&, + configureExecutionBurst_cb) override { + return Void(); + } + Return executeFenced(const V1_3::Request&, const hidl_vec&, + V1_2::MeasureTiming, const V1_3::OptionalTimePoint&, + const V1_3::OptionalTimeoutDuration&, + const V1_3::OptionalTimeoutDuration&, executeFenced_cb) override { + return Void(); + } +}; + +} // namespace + +class MemoryDomainTestBase : public testing::Test { + protected: + MemoryDomainTestBase(sp device, TestOperandType type) + : kDevice(std::move(device)), + kTestOperandType(type), + kTestOperand(kTestOperandMap.at(type)), + kTestOperandDataSize(nn::nonExtensionOperandSizeOfData(static_cast(type), + kTestOperand.dimensions)) {} + + void SetUp() override { + testing::Test::SetUp(); + ASSERT_NE(kDevice, nullptr); + } + + sp createConvPreparedModel(const TestOperand& testOperand, + uint32_t numOperations = 1) { + const TestModel testModel = createConvModel(testOperand, numOperations); + const Model model = createModel(testModel); + sp preparedModel; + createPreparedModel(kDevice, model, &preparedModel, /*reportSkipping=*/false); + return preparedModel; + } + + sp createAddPreparedModel(const TestOperand& testOperand) { + const TestModel testModel = createSingleAddModel(testOperand); + const Model model = createModel(testModel); + sp preparedModel; + createPreparedModel(kDevice, model, &preparedModel, /*reportSkipping=*/false); + return preparedModel; + } + + static const std::map kTestOperandMap; + + const sp kDevice; + const TestOperandType kTestOperandType; + const TestOperand& kTestOperand; + const uint32_t kTestOperandDataSize; +}; + +const std::map MemoryDomainTestBase::kTestOperandMap = { + {TestOperandType::TENSOR_FLOAT32, + { + .type = TestOperandType::TENSOR_FLOAT32, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_FLOAT16, + { + .type = TestOperandType::TENSOR_FLOAT16, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_QUANT8_ASYMM, + { + .type = TestOperandType::TENSOR_QUANT8_ASYMM, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.5f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, + { + .type = TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.5f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, +}; + +using MemoryDomainAllocateTestParam = std::tuple; +class MemoryDomainAllocateTest : public MemoryDomainTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainAllocateTest() + : MemoryDomainTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} + + struct AllocateTestArgs { + hidl_vec dimensions; + hidl_vec> preparedModels; + hidl_vec inputRoles; + hidl_vec outputRoles; + }; + + // Validation test for IDevice::allocate. The driver is expected to fail with INVALID_ARGUMENT, + // or GENERAL_FAILURE if memory domain is not supported. + void validateAllocate(AllocateTestArgs args) { + const auto ret = kDevice->allocate( + {.dimensions = std::move(args.dimensions)}, std::move(args.preparedModels), + std::move(args.inputRoles), std::move(args.outputRoles), + [](ErrorStatus status, const sp& buffer, uint32_t token) { + EXPECT_TRUE(status == ErrorStatus::INVALID_ARGUMENT || + status == ErrorStatus::GENERAL_FAILURE); + EXPECT_EQ(buffer, nullptr); + EXPECT_EQ(token, 0); + }); + ASSERT_TRUE(ret.isOk()); + } + + void testConflictOperands(const sp& model1, const sp& model2) { + validateAllocate({ + .preparedModels = {model1, model2}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {model1, model2}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + .outputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {model1, model2}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + } +}; + +TEST_P(MemoryDomainAllocateTest, EmptyRole) { + // Test with empty prepared models and roles. + validateAllocate({}); + + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // Test again with non-empty prepared models but empty roles. + validateAllocate({ + .preparedModels = {preparedModel}, + }); +} + +TEST_P(MemoryDomainAllocateTest, NullptrPreparedModel) { + // Test with nullptr prepared model as input role. + validateAllocate({ + .preparedModels = {nullptr}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Test with nullptr prepared model as output role. + validateAllocate({ + .preparedModels = {nullptr}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidPreparedModel) { + sp invalidPreparedModel = new InvalidPreparedModel(); + + // Test with invalid prepared model as input role. + validateAllocate({ + .preparedModels = {invalidPreparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Test with invalid prepared model as output role. + validateAllocate({ + .preparedModels = {invalidPreparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidModelIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the model index is out of bound. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // This should fail, because the model index is out of bound. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidIOIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the model only has one input. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 1, .frequency = 1.0f}}, + }); + + // This should fail, because the model only has one output. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 1, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidFrequency) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + for (float invalidFreq : {10.0f, 0.0f, -0.5f}) { + // Test with invalid frequency for input roles. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = invalidFreq}}, + }); + // Test with invalid frequency for output roles. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = invalidFreq}}, + }); + } +} + +TEST_P(MemoryDomainAllocateTest, SameRoleSpecifiedTwice) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // Same role with same model index. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Different model indexes, but logically referring to the same role. + validateAllocate({ + .preparedModels = {preparedModel, preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {preparedModel, preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictOperandType) { + const std::map conflictTypeMap = { + {TestOperandType::TENSOR_FLOAT32, TestOperandType::TENSOR_FLOAT16}, + {TestOperandType::TENSOR_FLOAT16, TestOperandType::TENSOR_FLOAT32}, + {TestOperandType::TENSOR_QUANT8_ASYMM, TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED}, + {TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, TestOperandType::TENSOR_QUANT8_ASYMM}, + }; + + TestOperand conflictTestOperand = kTestOperand; + const auto it = conflictTypeMap.find(kTestOperandType); + ASSERT_FALSE(it == conflictTypeMap.end()); + conflictTestOperand.type = it->second; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictScale) { + if (isFloat(kTestOperandType)) return; + + TestOperand conflictTestOperand = kTestOperand; + ASSERT_NE(conflictTestOperand.scale, 1.0f); + conflictTestOperand.scale = 1.0f; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictZeroPoint) { + if (isFloat(kTestOperandType)) return; + + TestOperand conflictTestOperand = kTestOperand; + ASSERT_NE(conflictTestOperand.zeroPoint, 10); + conflictTestOperand.zeroPoint = 10; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankBetweenRoles) { + TestOperand conflictTestOperand = kTestOperand; + conflictTestOperand.dimensions.pop_back(); + + auto preparedModel = createAddPreparedModel(kTestOperand); + auto conflictPreparedModel = createAddPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictDimensionsBetweenRoles) { + TestOperand conflictTestOperand = kTestOperand; + conflictTestOperand.dimensions[0] = 4; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankBetweenRoleAndDesc) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + auto badDimensions = kTestOperand.dimensions; + badDimensions.pop_back(); + + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictDimensionsBetweenRoleAndDesc) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + auto badDimensions = kTestOperand.dimensions; + badDimensions[0] = 4; + + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankWithScalarRole) { + auto preparedModel = createAddPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the target operand is a scalar but a non-empty dimension is + // specified. + validateAllocate({ + .dimensions = {1}, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 2, .frequency = 1.0f}}, + }); +} + +std::string printMemoryDomainAllocateTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType] = info.param; + const std::string type = toString(static_cast(operandType)); + return gtestCompliantName(getName(namedDevice) + "_" + type); +} + +INSTANTIATE_TEST_CASE_P(TestMemoryDomain, MemoryDomainAllocateTest, + testing::Combine(kNamedDeviceChoices, kTestOperandTypeChoices), + printMemoryDomainAllocateTest); + +class MemoryDomainCopyTestBase : public MemoryDomainTestBase { + protected: + MemoryDomainCopyTestBase(sp device, TestOperandType type) + : MemoryDomainTestBase(std::move(device), type) {} + + // Allocates device memory for roles of a single prepared model. + // Returns {IBuffer, token} if success; returns {nullptr, 0} if not supported. + std::pair, uint32_t> allocateBuffer(const sp& preparedModel, + const std::vector& inputIndexes, + const std::vector& outputIndexes, + const std::vector& dimensions) { + if (preparedModel == nullptr) { + return {nullptr, 0}; + } + + hidl_vec inputRoles(inputIndexes.size()), outputRoles(outputIndexes.size()); + auto trans = [](uint32_t ind) -> BufferRole { + return {.modelIndex = 0, .ioIndex = ind, .frequency = 1.0f}; + }; + std::transform(inputIndexes.begin(), inputIndexes.end(), inputRoles.begin(), trans); + std::transform(outputIndexes.begin(), outputIndexes.end(), outputRoles.begin(), trans); + + sp buffer; + uint32_t token = 0; + const auto ret = kDevice->allocate( + {.dimensions = dimensions}, {preparedModel}, std::move(inputRoles), + std::move(outputRoles), + [&buffer, &token](ErrorStatus err, const sp& buf, uint32_t tok) { + if (err == ErrorStatus::NONE) { + EXPECT_NE(buf, nullptr); + EXPECT_GT(tok, 0); + buffer = buf; + token = tok; + } else { + EXPECT_EQ(err, ErrorStatus::GENERAL_FAILURE); + EXPECT_EQ(buf, nullptr); + EXPECT_EQ(tok, 0); + } + }); + EXPECT_TRUE(ret.isOk()); + return {std::move(buffer), token}; + } + + std::pair, uint32_t> allocateBuffer(const sp& preparedModel, + const std::vector& inputIndexes, + const std::vector& outputIndexes) { + return allocateBuffer(preparedModel, inputIndexes, outputIndexes, {}); + } + + hidl_memory allocateSharedMemory(uint32_t size) { + hidl_memory memory = nn::allocateSharedMemory(size); + EXPECT_EQ(memory.size(), size); + return memory; + } + + void testCopyFrom(const sp& buffer, const hidl_memory& memory, + const std::vector& dimensions, ErrorStatus expectedStatus) { + const auto ret = buffer->copyFrom(memory, dimensions); + ASSERT_TRUE(ret.isOk()); + ASSERT_EQ(static_cast(ret), expectedStatus); + } + + void testCopyTo(const sp& buffer, const hidl_memory& memory, + ErrorStatus expectedStatus) { + const auto ret = buffer->copyTo(memory); + ASSERT_TRUE(ret.isOk()); + ASSERT_EQ(static_cast(ret), expectedStatus); + } + + void initializeDeviceMemory(const sp& buffer) { + hidl_memory memory = nn::allocateSharedMemory(kTestOperandDataSize); + ASSERT_EQ(memory.size(), kTestOperandDataSize); + testCopyFrom(buffer, memory, kTestOperand.dimensions, ErrorStatus::NONE); + } +}; + +using MemoryDomainCopyTestParam = std::tuple; +class MemoryDomainCopyTest : public MemoryDomainCopyTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainCopyTest() + : MemoryDomainCopyTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} +}; + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidMemorySize) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + hidl_memory badMemory1 = allocateSharedMemory(badMemorySize1); + hidl_memory badMemory2 = allocateSharedMemory(badMemorySize2); + testCopyFrom(buffer, badMemory1, {}, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, badMemory2, {}, ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidMemorySize_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + hidl_memory badMemory1 = allocateSharedMemory(badMemorySize1); + hidl_memory badMemory2 = allocateSharedMemory(badMemorySize2); + hidl_memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + auto badDimensions = kTestOperand.dimensions; + badDimensions[0] = 2; + + testCopyFrom(buffer, badMemory1, kTestOperand.dimensions, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, badMemory2, kTestOperand.dimensions, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, goodMemory, kTestOperand.dimensions, ErrorStatus::NONE); + testCopyFrom(buffer, goodMemory, badDimensions, ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidDimensions) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + hidl_memory memory = allocateSharedMemory(kTestOperandDataSize); + + std::vector badDimensions; + badDimensions = kTestOperand.dimensions; + badDimensions.pop_back(); + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = kTestOperand.dimensions; + badDimensions[0] = 2; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = kTestOperand.dimensions; + badDimensions[0] = 0; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + testCopyFrom(buffer, memory, {}, ErrorStatus::NONE); + testCopyFrom(buffer, memory, kTestOperand.dimensions, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidDimensions_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + hidl_memory memory = allocateSharedMemory(kTestOperandDataSize); + + std::vector badDimensions; + badDimensions = kTestOperand.dimensions; + badDimensions.pop_back(); + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = kTestOperand.dimensions; + badDimensions[0] = 2; + badDimensions[3] = 4; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = kTestOperand.dimensions; + badDimensions[0] = 1; + badDimensions[3] = 0; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + testCopyFrom(buffer, memory, {}, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, memory, kTestOperand.dimensions, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_UninitializedMemory) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + hidl_memory memory = allocateSharedMemory(kTestOperandDataSize); + testCopyTo(buffer, memory, ErrorStatus::GENERAL_FAILURE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_InvalidMemorySize) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + hidl_memory badMemory1 = allocateSharedMemory(badMemorySize1); + hidl_memory badMemory2 = allocateSharedMemory(badMemorySize2); + hidl_memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + initializeDeviceMemory(buffer); + testCopyTo(buffer, badMemory1, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, badMemory2, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, goodMemory, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_InvalidMemorySize_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + hidl_memory badMemory1 = allocateSharedMemory(badMemorySize1); + hidl_memory badMemory2 = allocateSharedMemory(badMemorySize2); + hidl_memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + initializeDeviceMemory(buffer); + testCopyTo(buffer, badMemory1, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, badMemory2, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, goodMemory, ErrorStatus::NONE); +} + +std::string printMemoryDomainCopyTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType] = info.param; + const std::string type = toString(static_cast(operandType)); + return gtestCompliantName(getName(namedDevice) + "_" + type); +} + +INSTANTIATE_TEST_CASE_P(TestMemoryDomain, MemoryDomainCopyTest, + testing::Combine(kNamedDeviceChoices, kTestOperandTypeChoices), + printMemoryDomainCopyTest); + +using MemoryDomainExecutionTestParam = std::tuple; +class MemoryDomainExecutionTest + : public MemoryDomainCopyTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainExecutionTest() + : MemoryDomainCopyTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} + + Request::MemoryPool createSharedMemoryPool(uint32_t size) { + hidl_memory memory = allocateSharedMemory(size); + Request::MemoryPool pool; + pool.hidlMemory(memory); + return pool; + } + + Request::MemoryPool createDeviceMemoryPool(uint32_t token) { + Request::MemoryPool pool; + pool.token(token); + return pool; + } + + void testExecution(const sp& preparedModel, const Request& request, + ErrorStatus expectedStatus) { + switch (kExecutor) { + case Executor::ASYNC: + EXPECT_EQ(executeAsync(preparedModel, request), expectedStatus); + break; + case Executor::SYNC: + EXPECT_EQ(executeSync(preparedModel, request), expectedStatus); + break; + default: + ASSERT_TRUE(false); + } + } + + ErrorStatus executeAsync(const sp& preparedModel, const Request& request) { + ErrorStatus executionStatus; + + // launch execution + sp executionCallback = new ExecutionCallback(); + const auto ret = + preparedModel->execute_1_3(request, MeasureTiming::NO, {}, {}, executionCallback); + EXPECT_TRUE(ret.isOk()); + executionStatus = static_cast(ret); + + // retrieve execution status + executionCallback->wait(); + if (executionStatus == ErrorStatus::NONE) { + executionStatus = executionCallback->getStatus(); + } else { + EXPECT_EQ(executionStatus, executionCallback->getStatus()); + } + const auto timing = executionCallback->getTiming(); + EXPECT_EQ(UINT64_MAX, timing.timeOnDevice); + EXPECT_EQ(UINT64_MAX, timing.timeInDriver); + if (executionStatus != ErrorStatus::NONE) { + EXPECT_EQ(executionCallback->getOutputShapes().size(), 0); + } + return executionStatus; + } + + ErrorStatus executeSync(const sp& preparedModel, const Request& request) { + ErrorStatus executionStatus; + const auto ret = preparedModel->executeSynchronously_1_3( + request, MeasureTiming::NO, {}, {}, + [&executionStatus](ErrorStatus error, const hidl_vec& shapes, + const Timing& time) { + executionStatus = error; + EXPECT_EQ(UINT64_MAX, time.timeOnDevice); + EXPECT_EQ(UINT64_MAX, time.timeInDriver); + if (executionStatus != ErrorStatus::NONE) { + EXPECT_EQ(shapes.size(), 0); + } + }); + EXPECT_TRUE(ret.isOk()); + return executionStatus; + } + + // TODO(xusongw): Add executeFenced. + + const Executor kExecutor = std::get(GetParam()); +}; + +TEST_P(MemoryDomainExecutionTest, InvalidToken) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + Request::MemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool badDeviceMemory1 = createDeviceMemoryPool(0); // Invalid token. + Request::MemoryPool badDeviceMemory2 = createDeviceMemoryPool(100); // Unknown token. + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, badDeviceMemory1}}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, badDeviceMemory2}}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, badDeviceMemory1}}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, badDeviceMemory2}}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidPreparedModel) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + auto badPreparedModel = createConvPreparedModel(kTestOperand); + if (badPreparedModel == nullptr) return; + + Request::MemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the buffer is not allocated for badPreparedModel. + initializeDeviceMemory(buffer); + testExecution(badPreparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(badPreparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidIOIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand, 2); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {}); + if (buffer == nullptr) return; + + Request::MemoryPool sharedMemory1 = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool sharedMemory2 = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool sharedMemory3 = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg1 = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg2 = { + .location = {.poolIndex = 1, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg3 = { + .location = {.poolIndex = 2, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 3}}; + + // This should fail, because the device memory is not allocated for input 1. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, deviceMemoryArg}, + .outputs = {sharedMemoryArg2, sharedMemoryArg3}, + .pools = {sharedMemory1, sharedMemory2, sharedMemory3, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the device memory is not allocated for output 1. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, sharedMemoryArg2}, + .outputs = {sharedMemoryArg3, deviceMemoryArg}, + .pools = {sharedMemory1, sharedMemory2, sharedMemory3, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidIOType) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [inputBuffer, inputToken] = allocateBuffer(preparedModel, {0}, {}); + auto [outputBuffer, outputToken] = allocateBuffer(preparedModel, {}, {0}); + if (inputBuffer == nullptr || outputBuffer == nullptr) return; + + Request::MemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(inputToken); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the device memory is allocated for input but used as output. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the device memory is allocated for output but used as input. + deviceMemory.token(outputToken); + initializeDeviceMemory(outputBuffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, UninitializedMemory) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + Request::MemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the device memory is not initialized. + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::GENERAL_FAILURE); + + // This should initialize the device memory. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::NONE); + + // Test again with initialized device memory. + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::NONE); +} + +TEST_P(MemoryDomainExecutionTest, SameRequestMultipleRoles) { + auto preparedModel = createConvPreparedModel(kTestOperand, 2); + auto [buffer, token] = allocateBuffer(preparedModel, {0, 1}, {0, 1}); + if (buffer == nullptr) return; + + Request::MemoryPool sharedMemory1 = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool sharedMemory2 = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg1 = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg2 = { + .location = {.poolIndex = 1, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 2}}; + + // This should fail, because the same device memory cannot be used for both input and output. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg, sharedMemoryArg1}, + .outputs = {deviceMemoryArg, sharedMemoryArg2}, + .pools = {sharedMemory1, sharedMemory2, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the same device memory cannot be used for multiple outputs. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, sharedMemoryArg2}, + .outputs = {deviceMemoryArg, deviceMemoryArg}, + .pools = {sharedMemory1, sharedMemory2, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + // The same device memory can be used for multiple inputs. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg, deviceMemoryArg}, + .outputs = {sharedMemoryArg1, sharedMemoryArg2}, + .pools = {sharedMemory1, sharedMemory2, deviceMemory}}, + ErrorStatus::NONE); +} + +TEST_P(MemoryDomainExecutionTest, InvalidDimensions) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}, kTestOperand.dimensions); + if (buffer == nullptr) return; + + Request::MemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + Request::MemoryPool deviceMemory = createDeviceMemoryPool(token); + auto badDimensions = kTestOperand.dimensions; + badDimensions[0] = 2; + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}, + .dimensions = badDimensions}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + RequestArgument deviceMemoryArgWithBadDimensions = {.location = {.poolIndex = 1}, + .dimensions = badDimensions}; + + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArgWithBadDimensions}, + .outputs = {sharedMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArgWithBadDimensions}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::INVALID_ARGUMENT); + + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = {sharedMemory, deviceMemory}}, + ErrorStatus::GENERAL_FAILURE); +} + +const auto kExecutorChoices = testing::Values(Executor::ASYNC, Executor::SYNC); + +std::string printMemoryDomainExecutionTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType, executor] = info.param; + const std::string type = toString(static_cast(operandType)); + const std::string executorStr = toString(executor); + return gtestCompliantName(getName(namedDevice) + "_" + type + "_" + executorStr); +} + +INSTANTIATE_TEST_CASE_P(TestMemoryDomain, MemoryDomainExecutionTest, + testing::Combine(kNamedDeviceChoices, kTestOperandTypeChoices, + kExecutorChoices), + printMemoryDomainExecutionTest); + +} // namespace android::hardware::neuralnetworks::V1_3::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp index 5b07034296..60ceb7e21e 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp @@ -192,4 +192,19 @@ sp getPreparedModel_1_3(const sp& callbac return IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr); } +std::string toString(Executor executor) { + switch (executor) { + case Executor::ASYNC: + return "ASYNC"; + case Executor::SYNC: + return "SYNC"; + case Executor::BURST: + return "BURST"; + case Executor::FENCED: + return "FENCED"; + default: + CHECK(false); + } +} + } // namespace android::hardware::neuralnetworks::V1_3::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h index 4e51052966..de082c39cc 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h @@ -52,6 +52,10 @@ void createPreparedModel(const sp& device, const Model& model, // Utility function to get PreparedModel from callback and downcast to V1_2. sp getPreparedModel_1_3(const sp& callback); +enum class Executor { ASYNC, SYNC, BURST, FENCED }; + +std::string toString(Executor executor); + } // namespace android::hardware::neuralnetworks::V1_3::vts::functional #endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_3_VTS_HAL_NEURALNETWORKS_H From 2844165bf3806404ab8ce2aa899c79b0a8eb647e Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Fri, 20 Mar 2020 14:49:54 -0700 Subject: [PATCH 2/5] Add fenced compute path to memory domain validation test. Bug: 147777318 Test: 1.3 VTS Change-Id: I0b731d10384ef2024241af1d908acf3ba760d73f Merged-In: I0b731d10384ef2024241af1d908acf3ba760d73f (cherry picked from commit 9c415917e0859f42bc4e6d020aa665c0e48fc6dc) --- .../vts/functional/GeneratedTestHarness.cpp | 14 +++---- .../1.3/vts/functional/GeneratedTestHarness.h | 2 + .../1.3/vts/functional/MemoryDomainTests.cpp | 41 ++++++++++++++++++- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index ff21960e6a..29fdee4398 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -78,13 +78,6 @@ enum class MemoryType { SHARED, DEVICE }; enum class IOType { INPUT, OUTPUT }; -static void waitForSyncFence(int syncFd) { - constexpr int kInfiniteTimeout = -1; - ASSERT_GT(syncFd, 0); - int r = sync_wait(syncFd, kInfiniteTimeout); - ASSERT_GE(r, 0); -} - struct TestConfig { Executor executor; MeasureTiming measureTiming; @@ -275,6 +268,13 @@ void copyTestBuffers(const std::vector& buffers, uint8_t* out } // namespace +void waitForSyncFence(int syncFd) { + constexpr int kInfiniteTimeout = -1; + ASSERT_GT(syncFd, 0); + int r = sync_wait(syncFd, kInfiniteTimeout); + ASSERT_GE(r, 0); +} + Model createModel(const TestModel& testModel) { uint32_t constCopySize = 0; uint32_t constRefSize = 0; diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h index 834d335f50..38d6486478 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h @@ -77,6 +77,8 @@ enum class TestKind { void EvaluatePreparedModel(const sp& device, const sp& preparedModel, const test_helper::TestModel& testModel, TestKind testKind); +void waitForSyncFence(int syncFd); + } // namespace android::hardware::neuralnetworks::V1_3::vts::functional #endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H diff --git a/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp b/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp index 08c1b35510..3c0c8858b6 100644 --- a/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp +++ b/neuralnetworks/1.3/vts/functional/MemoryDomainTests.cpp @@ -864,6 +864,9 @@ class MemoryDomainExecutionTest case Executor::SYNC: EXPECT_EQ(executeSync(preparedModel, request), expectedStatus); break; + case Executor::FENCED: + EXPECT_EQ(executeFenced(preparedModel, request), expectedStatus); + break; default: ASSERT_TRUE(false); } @@ -912,7 +915,38 @@ class MemoryDomainExecutionTest return executionStatus; } - // TODO(xusongw): Add executeFenced. + ErrorStatus executeFenced(const sp& preparedModel, const Request& request) { + ErrorStatus executionStatus; + hidl_handle syncFenceHandle; + sp fencedCallback; + const auto callbackFunc = [&executionStatus, &syncFenceHandle, &fencedCallback]( + ErrorStatus error, const hidl_handle& handle, + const sp& callback) { + executionStatus = error; + syncFenceHandle = handle; + fencedCallback = callback; + }; + Return ret = preparedModel->executeFenced(request, {}, MeasureTiming::NO, {}, {}, {}, + callbackFunc); + EXPECT_TRUE(ret.isOk()); + if (executionStatus != ErrorStatus::NONE) { + EXPECT_EQ(syncFenceHandle.getNativeHandle(), nullptr); + EXPECT_EQ(fencedCallback, nullptr); + return executionStatus; + } + if (syncFenceHandle.getNativeHandle()) { + waitForSyncFence(syncFenceHandle.getNativeHandle()->data[0]); + } + EXPECT_NE(fencedCallback, nullptr); + ret = fencedCallback->getExecutionInfo( + [&executionStatus](ErrorStatus error, Timing t, Timing) { + executionStatus = error; + EXPECT_EQ(UINT64_MAX, t.timeOnDevice); + EXPECT_EQ(UINT64_MAX, t.timeInDriver); + }); + EXPECT_TRUE(ret.isOk()); + return executionStatus; + } const Executor kExecutor = std::get(GetParam()); }; @@ -1111,6 +1145,9 @@ TEST_P(MemoryDomainExecutionTest, SameRequestMultipleRoles) { } TEST_P(MemoryDomainExecutionTest, InvalidDimensions) { + // FENCED execution does not support dynamic shape. + if (kExecutor == Executor::FENCED) return; + TestOperand testOperand = kTestOperand; testOperand.dimensions[0] = 0; auto preparedModel = createConvPreparedModel(testOperand); @@ -1148,7 +1185,7 @@ TEST_P(MemoryDomainExecutionTest, InvalidDimensions) { ErrorStatus::GENERAL_FAILURE); } -const auto kExecutorChoices = testing::Values(Executor::ASYNC, Executor::SYNC); +const auto kExecutorChoices = testing::Values(Executor::ASYNC, Executor::SYNC, Executor::FENCED); std::string printMemoryDomainExecutionTest( const testing::TestParamInfo& info) { From 75e63ad74354c36d7e76bdda21f14d64af1a6110 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Tue, 25 Feb 2020 11:43:10 -0800 Subject: [PATCH 3/5] Add BLOB AHWB tests in VTS. Bug: 149847930 Test: 1.3 VTS Change-Id: I9c795dcb7696c843afd12551927463c5529a4b60 Merged-In: I9c795dcb7696c843afd12551927463c5529a4b60 (cherry picked from commit 41adc5bc115d558cecfdb933d8dae7b17f707b20) --- .../vts/functional/GeneratedTestHarness.cpp | 6 +- neuralnetworks/1.0/vts/functional/Utils.cpp | 91 ++++++++-- .../vts/functional/VtsHalNeuralnetworks.cpp | 3 +- .../1.0/vts/functional/include/1.0/Utils.h | 72 +++++++- .../vts/functional/GeneratedTestHarness.cpp | 6 +- .../vts/functional/VtsHalNeuralnetworks.cpp | 3 +- .../vts/functional/GeneratedTestHarness.cpp | 20 ++- .../vts/functional/VtsHalNeuralnetworks.cpp | 3 +- .../vts/functional/GeneratedTestHarness.cpp | 164 ++++++++++-------- .../vts/functional/QualityOfServiceTests.cpp | 15 +- .../vts/functional/VtsHalNeuralnetworks.cpp | 3 +- 11 files changed, 275 insertions(+), 111 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index e28605dca2..4ab228f85b 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -125,7 +125,9 @@ Model createModel(const TestModel& testModel) { // Test driver for those generated from ml/nn/runtime/test/spec void Execute(const sp& device, const TestModel& testModel) { const Model model = createModel(testModel); - const Request request = createRequest(testModel); + + ExecutionContext context; + const Request request = context.createRequest(testModel); // Create IPreparedModel. sp preparedModel; @@ -143,7 +145,7 @@ void Execute(const sp& device, const TestModel& testModel) { ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus()); // Retrieve execution results. - const std::vector outputs = getOutputBuffers(request); + const std::vector outputs = context.getOutputBuffers(request); // We want "close-enough" results. checkResults(testModel, outputs); diff --git a/neuralnetworks/1.0/vts/functional/Utils.cpp b/neuralnetworks/1.0/vts/functional/Utils.cpp index 0dba85acd9..3613e69088 100644 --- a/neuralnetworks/1.0/vts/functional/Utils.cpp +++ b/neuralnetworks/1.0/vts/functional/Utils.cpp @@ -21,10 +21,13 @@ #include #include +#include #include #include #include +#include +#include #include #include #include @@ -37,10 +40,64 @@ using V1_0::DataLocation; using V1_0::Request; using V1_0::RequestArgument; -constexpr uint32_t kInputPoolIndex = 0; -constexpr uint32_t kOutputPoolIndex = 1; +std::unique_ptr TestAshmem::create(uint32_t size) { + auto ashmem = std::make_unique(size); + return ashmem->mIsValid ? std::move(ashmem) : nullptr; +} + +void TestAshmem::initialize(uint32_t size) { + mIsValid = false; + ASSERT_GT(size, 0); + mHidlMemory = nn::allocateSharedMemory(size); + ASSERT_TRUE(mHidlMemory.valid()); + mMappedMemory = mapMemory(mHidlMemory); + ASSERT_NE(mMappedMemory, nullptr); + mPtr = static_cast(static_cast(mMappedMemory->getPointer())); + ASSERT_NE(mPtr, nullptr); + mIsValid = true; +} + +std::unique_ptr TestBlobAHWB::create(uint32_t size) { + auto ahwb = std::make_unique(size); + return ahwb->mIsValid ? std::move(ahwb) : nullptr; +} + +void TestBlobAHWB::initialize(uint32_t size) { + mIsValid = false; + ASSERT_GT(size, 0); + const auto usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN; + const AHardwareBuffer_Desc desc = { + .width = size, + .height = 1, + .layers = 1, + .format = AHARDWAREBUFFER_FORMAT_BLOB, + .usage = usage, + .stride = size, + }; + ASSERT_EQ(AHardwareBuffer_allocate(&desc, &mAhwb), 0); + ASSERT_NE(mAhwb, nullptr); + + void* buffer = nullptr; + ASSERT_EQ(AHardwareBuffer_lock(mAhwb, usage, -1, nullptr, &buffer), 0); + ASSERT_NE(buffer, nullptr); + mPtr = static_cast(buffer); + + const native_handle_t* handle = AHardwareBuffer_getNativeHandle(mAhwb); + ASSERT_NE(handle, nullptr); + mHidlMemory = hidl_memory("hardware_buffer_blob", handle, desc.width); + mIsValid = true; +} + +TestBlobAHWB::~TestBlobAHWB() { + if (mAhwb) { + AHardwareBuffer_unlock(mAhwb, nullptr); + AHardwareBuffer_release(mAhwb); + } +} + +Request ExecutionContext::createRequest(const TestModel& testModel, MemoryType memoryType) { + CHECK(memoryType == MemoryType::ASHMEM || memoryType == MemoryType::BLOB_AHWB); -Request createRequest(const TestModel& testModel) { // Model inputs. hidl_vec inputs(testModel.main.inputIndexes.size()); size_t inputSize = 0; @@ -80,16 +137,19 @@ Request createRequest(const TestModel& testModel) { } // Allocate memory pools. - hidl_vec pools = {nn::allocateSharedMemory(inputSize), - nn::allocateSharedMemory(outputSize)}; - CHECK_NE(pools[kInputPoolIndex].size(), 0u); - CHECK_NE(pools[kOutputPoolIndex].size(), 0u); - sp inputMemory = mapMemory(pools[kInputPoolIndex]); - CHECK(inputMemory.get() != nullptr); - uint8_t* inputPtr = static_cast(static_cast(inputMemory->getPointer())); - CHECK(inputPtr != nullptr); + if (memoryType == MemoryType::ASHMEM) { + mInputMemory = TestAshmem::create(inputSize); + mOutputMemory = TestAshmem::create(outputSize); + } else { + mInputMemory = TestBlobAHWB::create(inputSize); + mOutputMemory = TestBlobAHWB::create(outputSize); + } + EXPECT_NE(mInputMemory, nullptr); + EXPECT_NE(mOutputMemory, nullptr); + hidl_vec pools = {mInputMemory->getHidlMemory(), mOutputMemory->getHidlMemory()}; // Copy input data to the memory pool. + uint8_t* inputPtr = mInputMemory->getPointer(); for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; if (op.data.size() > 0) { @@ -102,18 +162,13 @@ Request createRequest(const TestModel& testModel) { return {.inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)}; } -std::vector getOutputBuffers(const Request& request) { - sp outputMemory = mapMemory(request.pools[kOutputPoolIndex]); - CHECK(outputMemory.get() != nullptr); - uint8_t* outputPtr = static_cast(static_cast(outputMemory->getPointer())); - CHECK(outputPtr != nullptr); - +std::vector ExecutionContext::getOutputBuffers(const Request& request) const { // Copy out output results. + uint8_t* outputPtr = mOutputMemory->getPointer(); std::vector outputBuffers; for (const auto& output : request.outputs) { outputBuffers.emplace_back(output.location.length, outputPtr + output.location.offset); } - return outputBuffers; } diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp index cb2225025b..7f7dac056b 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp @@ -129,7 +129,8 @@ void validateEverything(const sp& device, const Model& model, const Req TEST_P(ValidationTest, Test) { const Model model = createModel(kTestModel); - const Request request = createRequest(kTestModel); + ExecutionContext context; + const Request request = context.createRequest(kTestModel); ASSERT_FALSE(kTestModel.expectFailure); validateEverything(kDevice, model, request); } diff --git a/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h b/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h index 6d4534cb4e..3292f79b1a 100644 --- a/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h +++ b/neuralnetworks/1.0/vts/functional/include/1.0/Utils.h @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include #include @@ -28,11 +30,73 @@ namespace android::hardware::neuralnetworks { -// Create HIDL Request from the TestModel struct. -V1_0::Request createRequest(const test_helper::TestModel& testModel); +// Convenience class to manage the lifetime of memory resources. +class TestMemoryBase { + DISALLOW_COPY_AND_ASSIGN(TestMemoryBase); -// After execution, copy out output results from the output memory pool. -std::vector<::test_helper::TestBuffer> getOutputBuffers(const V1_0::Request& request); + public: + TestMemoryBase() = default; + virtual ~TestMemoryBase() = default; + uint8_t* getPointer() const { return mPtr; } + hidl_memory getHidlMemory() const { return mHidlMemory; } + + protected: + uint8_t* mPtr = nullptr; + hidl_memory mHidlMemory; + bool mIsValid = false; +}; + +class TestAshmem : public TestMemoryBase { + public: + static std::unique_ptr create(uint32_t size); + + // Prefer TestAshmem::create. + // The constructor calls initialize, which constructs the memory resources. This is a workaround + // that gtest macros cannot be used directly in a constructor. + TestAshmem(uint32_t size) { initialize(size); } + + private: + void initialize(uint32_t size); + sp mMappedMemory; +}; + +class TestBlobAHWB : public TestMemoryBase { + public: + static std::unique_ptr create(uint32_t size); + + // Prefer TestBlobAHWB::create. + // The constructor calls initialize, which constructs the memory resources. This is a + // workaround that gtest macros cannot be used directly in a constructor. + TestBlobAHWB(uint32_t size) { initialize(size); } + ~TestBlobAHWB(); + + private: + void initialize(uint32_t size); + AHardwareBuffer* mAhwb = nullptr; +}; + +enum class MemoryType { ASHMEM, BLOB_AHWB, DEVICE }; + +// Manages the lifetime of memory resources used in an execution. +class ExecutionContext { + DISALLOW_COPY_AND_ASSIGN(ExecutionContext); + + public: + static constexpr uint32_t kInputPoolIndex = 0; + static constexpr uint32_t kOutputPoolIndex = 1; + + ExecutionContext() = default; + + // Create HIDL Request from the TestModel struct. + V1_0::Request createRequest(const test_helper::TestModel& testModel, + MemoryType memoryType = MemoryType::ASHMEM); + + // After execution, copy out output results from the output memory pool. + std::vector getOutputBuffers(const V1_0::Request& request) const; + + private: + std::unique_ptr mInputMemory, mOutputMemory; +}; // Delete element from hidl_vec. hidl_vec doesn't support a "remove" operation, // so this is efficiently accomplished by moving the element to the end and diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp index cee15a35a1..14d300db71 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp @@ -133,7 +133,9 @@ Model createModel(const TestModel& testModel) { // Test driver for those generated from ml/nn/runtime/test/spec void Execute(const sp& device, const TestModel& testModel) { const Model model = createModel(testModel); - const Request request = createRequest(testModel); + + ExecutionContext context; + const Request request = context.createRequest(testModel); // Create IPreparedModel. sp preparedModel; @@ -151,7 +153,7 @@ void Execute(const sp& device, const TestModel& testModel) { ASSERT_EQ(ErrorStatus::NONE, executionCallback->getStatus()); // Retrieve execution results. - const std::vector outputs = getOutputBuffers(request); + const std::vector outputs = context.getOutputBuffers(request); // We want "close-enough" results. checkResults(testModel, outputs); diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp index d56d40b2ba..04af6ec704 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp @@ -132,7 +132,8 @@ void validateEverything(const sp& device, const Model& model, const Req TEST_P(ValidationTest, Test) { const Model model = createModel(kTestModel); - const Request request = createRequest(kTestModel); + ExecutionContext context; + const Request request = context.createRequest(kTestModel); ASSERT_FALSE(kTestModel.expectFailure); validateEverything(kDevice, model, request); } diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp index 3ab01351e9..aaaafc7b72 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp @@ -68,6 +68,7 @@ struct TestConfig { Executor executor; MeasureTiming measureTiming; OutputType outputType; + MemoryType memoryType; }; } // namespace @@ -216,7 +217,8 @@ void EvaluatePreparedModel(const sp& preparedModel, const TestMo return; } - Request request = createRequest(testModel); + ExecutionContext context; + Request request = context.createRequest(testModel, testConfig.memoryType); if (testConfig.outputType == OutputType::INSUFFICIENT) { makeOutputInsufficientSize(/*outputIndex=*/0, &request); } @@ -326,7 +328,7 @@ void EvaluatePreparedModel(const sp& preparedModel, const TestMo } // Retrieve execution results. - const std::vector outputs = getOutputBuffers(request); + const std::vector outputs = context.getOutputBuffers(request); // We want "close-enough" results. checkResults(testModel, outputs); @@ -337,24 +339,30 @@ void EvaluatePreparedModel(const sp& preparedModel, const TestMo std::vector outputTypesList; std::vector measureTimingList; std::vector executorList; + std::vector memoryTypeList; if (testDynamicOutputShape) { outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST}; + memoryTypeList = {MemoryType::ASHMEM}; } else { outputTypesList = {OutputType::FULLY_SPECIFIED}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST}; + memoryTypeList = {MemoryType::ASHMEM, MemoryType::BLOB_AHWB}; } for (const OutputType outputType : outputTypesList) { for (const MeasureTiming measureTiming : measureTimingList) { for (const Executor executor : executorList) { - const TestConfig testConfig = {.executor = executor, - .measureTiming = measureTiming, - .outputType = outputType}; - EvaluatePreparedModel(preparedModel, testModel, testConfig); + for (const MemoryType memoryType : memoryTypeList) { + const TestConfig testConfig = {.executor = executor, + .measureTiming = measureTiming, + .outputType = outputType, + .memoryType = memoryType}; + EvaluatePreparedModel(preparedModel, testModel, testConfig); + } } } } diff --git a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp index 4fbd0e270f..5853fa49f6 100644 --- a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp @@ -153,7 +153,8 @@ void validateFailure(const sp& device, const Model& model, const Reques TEST_P(ValidationTest, Test) { const Model model = createModel(kTestModel); - const Request request = createRequest(kTestModel); + ExecutionContext context; + const Request request = context.createRequest(kTestModel); if (kTestModel.expectFailure) { validateFailure(kDevice, model, request); } else { diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 29fdee4398..ff71778d83 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -74,8 +74,6 @@ namespace { enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE }; -enum class MemoryType { SHARED, DEVICE }; - enum class IOType { INPUT, OUTPUT }; struct TestConfig { @@ -336,21 +334,39 @@ static void makeOutputDimensionsUnspecified(Model* model) { } } -constexpr uint32_t kInputPoolIndex = 0; -constexpr uint32_t kOutputPoolIndex = 1; -constexpr uint32_t kDeviceMemoryBeginIndex = 2; +class ExecutionContextV1_3 { + public: + ExecutionContextV1_3(sp device, sp preparedModel) + : kDevice(std::move(device)), kPreparedModel(std::move(preparedModel)) {} -static std::pair>> createRequest( - const sp& device, const sp& preparedModel, - const TestModel& testModel, bool preferDeviceMemory) { + std::optional createRequest(const TestModel& testModel, MemoryType memoryType); + std::vector getOutputBuffers(const TestModel& testModel, + const Request& request) const; + + private: + // Get a TestBuffer with data copied from an IBuffer object. + void getBuffer(const sp& buffer, size_t size, TestBuffer* testBuffer) const; + + static constexpr uint32_t kInputPoolIndex = 0; + static constexpr uint32_t kOutputPoolIndex = 1; + static constexpr uint32_t kDeviceMemoryBeginIndex = 2; + + const sp kDevice; + const sp kPreparedModel; + std::unique_ptr mInputMemory, mOutputMemory; + std::vector> mBuffers; +}; + +std::optional ExecutionContextV1_3::createRequest(const TestModel& testModel, + MemoryType memoryType) { // Memory pools are organized as: // - 0: Input shared memory pool // - 1: Output shared memory pool // - [2, 2+i): Input device memories // - [2+i, 2+i+o): Output device memories - DeviceMemoryAllocator allocator(device, preparedModel, testModel); - std::vector> buffers; + DeviceMemoryAllocator allocator(kDevice, kPreparedModel, testModel); std::vector tokens; + mBuffers.clear(); // Model inputs. hidl_vec inputs(testModel.main.inputIndexes.size()); @@ -361,13 +377,13 @@ static std::pair>> createRequest( // Omitted input. inputs[i] = {.hasNoValue = true}; continue; - } else if (preferDeviceMemory) { + } else if (memoryType == MemoryType::DEVICE) { SCOPED_TRACE("Input index = " + std::to_string(i)); auto [buffer, token] = allocator.allocate(i); if (buffer != nullptr) { - DataLocation loc = {.poolIndex = static_cast(buffers.size() + + DataLocation loc = {.poolIndex = static_cast(mBuffers.size() + kDeviceMemoryBeginIndex)}; - buffers.push_back(std::move(buffer)); + mBuffers.push_back(std::move(buffer)); tokens.push_back(token); inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; continue; @@ -387,13 +403,13 @@ static std::pair>> createRequest( size_t outputSize = 0; for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) { const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; - if (preferDeviceMemory) { + if (memoryType == MemoryType::DEVICE) { SCOPED_TRACE("Output index = " + std::to_string(i)); auto [buffer, token] = allocator.allocate(i); if (buffer != nullptr) { - DataLocation loc = {.poolIndex = static_cast(buffers.size() + + DataLocation loc = {.poolIndex = static_cast(mBuffers.size() + kDeviceMemoryBeginIndex)}; - buffers.push_back(std::move(buffer)); + mBuffers.push_back(std::move(buffer)); tokens.push_back(token); outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; continue; @@ -416,21 +432,29 @@ static std::pair>> createRequest( outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; } + if (memoryType == MemoryType::DEVICE && mBuffers.empty()) { + return std::nullopt; + } + // Memory pools. - hidl_vec pools(kDeviceMemoryBeginIndex + buffers.size()); - pools[kInputPoolIndex].hidlMemory(nn::allocateSharedMemory(std::max(inputSize, 1))); - pools[kOutputPoolIndex].hidlMemory(nn::allocateSharedMemory(std::max(outputSize, 1))); - CHECK_NE(pools[kInputPoolIndex].hidlMemory().size(), 0u); - CHECK_NE(pools[kOutputPoolIndex].hidlMemory().size(), 0u); - for (uint32_t i = 0; i < buffers.size(); i++) { + hidl_vec pools(kDeviceMemoryBeginIndex + mBuffers.size()); + if (memoryType == MemoryType::BLOB_AHWB) { + mInputMemory = TestBlobAHWB::create(std::max(inputSize, 1)); + mOutputMemory = TestBlobAHWB::create(std::max(outputSize, 1)); + } else { + mInputMemory = TestAshmem::create(std::max(inputSize, 1)); + mOutputMemory = TestAshmem::create(std::max(outputSize, 1)); + } + EXPECT_NE(mInputMemory, nullptr); + EXPECT_NE(mOutputMemory, nullptr); + pools[kInputPoolIndex].hidlMemory(mInputMemory->getHidlMemory()); + pools[kOutputPoolIndex].hidlMemory(mOutputMemory->getHidlMemory()); + for (uint32_t i = 0; i < mBuffers.size(); i++) { pools[kDeviceMemoryBeginIndex + i].token(tokens[i]); } // Copy input data to the input shared memory pool. - sp inputMemory = mapMemory(pools[kInputPoolIndex].hidlMemory()); - CHECK(inputMemory.get() != nullptr); - uint8_t* inputPtr = static_cast(static_cast(inputMemory->getPointer())); - CHECK(inputPtr != nullptr); + uint8_t* inputPtr = mInputMemory->getPointer(); for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { if (!inputs[i].hasNoValue && inputs[i].location.poolIndex == kInputPoolIndex) { const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; @@ -439,14 +463,38 @@ static std::pair>> createRequest( std::copy(begin, end, inputPtr + inputs[i].location.offset); } } - - Request request = { + return Request{ .inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)}; - return {std::move(request), std::move(buffers)}; +} + +std::vector ExecutionContextV1_3::getOutputBuffers(const TestModel& testModel, + const Request& request) const { + // Copy out output results. + uint8_t* outputPtr = mOutputMemory->getPointer(); + std::vector outputBuffers; + for (uint32_t i = 0; i < request.outputs.size(); i++) { + const auto& outputLoc = request.outputs[i].location; + if (outputLoc.poolIndex == kOutputPoolIndex) { + outputBuffers.emplace_back(outputLoc.length, outputPtr + outputLoc.offset); + } else { + const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; + if (op.data.size() == 0) { + outputBuffers.emplace_back(0, nullptr); + } else { + SCOPED_TRACE("Output index = " + std::to_string(i)); + const uint32_t bufferIndex = outputLoc.poolIndex - kDeviceMemoryBeginIndex; + TestBuffer buffer; + getBuffer(mBuffers[bufferIndex], op.data.size(), &buffer); + outputBuffers.push_back(std::move(buffer)); + } + } + } + return outputBuffers; } // Get a TestBuffer with data copied from an IBuffer object. -static void getBuffer(const sp& buffer, size_t size, TestBuffer* testBuffer) { +void ExecutionContextV1_3::getBuffer(const sp& buffer, size_t size, + TestBuffer* testBuffer) const { // IBuffer -> Shared memory. hidl_memory tmp = nn::allocateSharedMemory(size); const auto ret = buffer->copyTo(tmp); @@ -462,35 +510,6 @@ static void getBuffer(const sp& buffer, size_t size, TestBuffer* testBu *testBuffer = TestBuffer(size, outputPtr); } -static std::vector getOutputBuffers(const TestModel& testModel, const Request& request, - const std::vector>& buffers) { - sp outputMemory = mapMemory(request.pools[kOutputPoolIndex].hidlMemory()); - CHECK(outputMemory.get() != nullptr); - uint8_t* outputPtr = static_cast(static_cast(outputMemory->getPointer())); - CHECK(outputPtr != nullptr); - - // Copy out output results. - std::vector outputBuffers; - for (uint32_t i = 0; i < request.outputs.size(); i++) { - const auto& outputLoc = request.outputs[i].location; - if (outputLoc.poolIndex == kOutputPoolIndex) { - outputBuffers.emplace_back(outputLoc.length, outputPtr + outputLoc.offset); - } else { - const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; - if (op.data.size() == 0) { - outputBuffers.emplace_back(); - } else { - SCOPED_TRACE("Output index = " + std::to_string(i)); - const uint32_t bufferIndex = outputLoc.poolIndex - kDeviceMemoryBeginIndex; - TestBuffer buffer; - getBuffer(buffers[bufferIndex], op.data.size(), &buffer); - outputBuffers.push_back(std::move(buffer)); - } - } - } - return outputBuffers; -} - static bool hasZeroSizedOutput(const TestModel& testModel) { return std::any_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(), [&testModel](uint32_t index) { @@ -541,13 +560,14 @@ void EvaluatePreparedModel(const sp& device, const sp& return; } - auto [request, buffers] = - createRequest(device, preparedModel, testModel, - /*preferDeviceMemory=*/testConfig.memoryType == MemoryType::DEVICE); + ExecutionContextV1_3 context(device, preparedModel); + auto maybeRequest = context.createRequest(testModel, testConfig.memoryType); // Skip if testing memory domain but no device memory has been allocated. - if (testConfig.memoryType == MemoryType::DEVICE && buffers.empty()) { + if (!maybeRequest.has_value()) { return; } + + Request request = std::move(maybeRequest.value()); if (testConfig.outputType == OutputType::INSUFFICIENT) { makeOutputInsufficientSize(/*outputIndex=*/0, &request); } @@ -742,7 +762,7 @@ void EvaluatePreparedModel(const sp& device, const sp& } // Retrieve execution results. - const std::vector outputs = getOutputBuffers(testModel, request, buffers); + const std::vector outputs = context.getOutputBuffers(testModel, request); // We want "close-enough" results. checkResults(testModel, outputs); @@ -753,29 +773,32 @@ void EvaluatePreparedModel(const sp& device, const sp& std::vector outputTypesList; std::vector measureTimingList; std::vector executorList; - MemoryType memoryType = MemoryType::SHARED; + std::vector memoryTypeList; switch (testKind) { case TestKind::GENERAL: { outputTypesList = {OutputType::FULLY_SPECIFIED}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST}; + memoryTypeList = {MemoryType::ASHMEM}; } break; case TestKind::DYNAMIC_SHAPE: { outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; executorList = {Executor::ASYNC, Executor::SYNC, Executor::BURST, Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; } break; case TestKind::MEMORY_DOMAIN: { outputTypesList = {OutputType::FULLY_SPECIFIED}; measureTimingList = {MeasureTiming::NO}; executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED}; - memoryType = MemoryType::DEVICE; + memoryTypeList = {MemoryType::BLOB_AHWB, MemoryType::DEVICE}; } break; case TestKind::FENCED_COMPUTE: { outputTypesList = {OutputType::FULLY_SPECIFIED}; measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; executorList = {Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; } break; case TestKind::QUANTIZATION_COUPLING: { LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel"; @@ -786,14 +809,17 @@ void EvaluatePreparedModel(const sp& device, const sp& measureTimingList = {MeasureTiming::NO, MeasureTiming::YES}; // Burst does not support V1_3 loop timeout. executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; } break; } for (const OutputType outputType : outputTypesList) { for (const MeasureTiming measureTiming : measureTimingList) { for (const Executor executor : executorList) { - const TestConfig testConfig(executor, measureTiming, outputType, memoryType); - EvaluatePreparedModel(device, preparedModel, testModel, testConfig); + for (const MemoryType memoryType : memoryTypeList) { + const TestConfig testConfig(executor, measureTiming, outputType, memoryType); + EvaluatePreparedModel(device, preparedModel, testModel, testConfig); + } } } } @@ -812,7 +838,7 @@ void EvaluatePreparedCoupledModels(const sp& device, for (const OutputType outputType : outputTypesList) { for (const MeasureTiming measureTiming : measureTimingList) { for (const Executor executor : executorList) { - const TestConfig testConfig(executor, measureTiming, outputType, MemoryType::SHARED, + const TestConfig testConfig(executor, measureTiming, outputType, MemoryType::ASHMEM, /*reportSkipping=*/false); bool baseSkipped = false; EvaluatePreparedModel(device, preparedModel, testModel, testConfig, &baseSkipped); diff --git a/neuralnetworks/1.3/vts/functional/QualityOfServiceTests.cpp b/neuralnetworks/1.3/vts/functional/QualityOfServiceTests.cpp index 879989ea2a..2ef1e8f6bd 100644 --- a/neuralnetworks/1.3/vts/functional/QualityOfServiceTests.cpp +++ b/neuralnetworks/1.3/vts/functional/QualityOfServiceTests.cpp @@ -214,7 +214,8 @@ static MaybeResults executeSynchronously(const sp& preparedModel } void runExecutionTest(const sp& preparedModel, const TestModel& testModel, - const Request& request, bool synchronous, DeadlineBoundType deadlineBound) { + const Request& request, const ExecutionContext& context, bool synchronous, + DeadlineBoundType deadlineBound) { const ExecutionFunction execute = synchronous ? executeSynchronously : executeAsynchronously; const auto deadline = makeDeadline(deadlineBound); @@ -261,7 +262,7 @@ void runExecutionTest(const sp& preparedModel, const TestModel& // Retrieve execution results. ASSERT_TRUE(nn::compliantWithV1_0(request)); const V1_0::Request request10 = nn::convertToV1_0(request); - const std::vector outputs = getOutputBuffers(request10); + const std::vector outputs = context.getOutputBuffers(request10); // We want "close-enough" results. if (status == ErrorStatus::NONE) { @@ -270,10 +271,11 @@ void runExecutionTest(const sp& preparedModel, const TestModel& } void runExecutionTests(const sp& preparedModel, const TestModel& testModel, - const Request& request) { + const Request& request, const ExecutionContext& context) { for (bool synchronous : {false, true}) { for (auto deadlineBound : deadlineBounds) { - runExecutionTest(preparedModel, testModel, request, synchronous, deadlineBound); + runExecutionTest(preparedModel, testModel, request, context, synchronous, + deadlineBound); } } } @@ -291,8 +293,9 @@ void runTests(const sp& device, const TestModel& testModel) { if (preparedModel == nullptr) return; // run execution tests - const Request request = nn::convertToV1_3(createRequest(testModel)); - runExecutionTests(preparedModel, testModel, request); + ExecutionContext context; + const Request request = nn::convertToV1_3(context.createRequest(testModel)); + runExecutionTests(preparedModel, testModel, request, context); } class DeadlineTest : public GeneratedTestBase {}; diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp index 60ceb7e21e..f7bd6241ad 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp @@ -177,7 +177,8 @@ void validateFailure(const sp& device, const Model& model, const Reques TEST_P(ValidationTest, Test) { const Model model = createModel(kTestModel); - const Request request = nn::convertToV1_3(createRequest(kTestModel)); + ExecutionContext context; + const Request request = nn::convertToV1_3(context.createRequest(kTestModel)); if (kTestModel.expectFailure) { validateFailure(kDevice, model, request); } else { From 702a91acf58a7740f5d9c965a8c519c24f7758f6 Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Fri, 20 Mar 2020 15:09:29 -0700 Subject: [PATCH 4/5] Fix the timing initialization error for failed executeFenced case Fixes: 152075771 Test: mm Test: VtsHalNeuralnetworksV1_3TargetTest Change-Id: I5829397346354ee3fc4a58b0a418197a1eee47cc Merged-In: I5829397346354ee3fc4a58b0a418197a1eee47cc (cherry picked from commit ad7678c8cb77372ff58c50b9ac142e828ef3764c) --- neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index ff71778d83..260fa3f326 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -666,6 +666,7 @@ void EvaluatePreparedModel(const sp& device, const sp& ASSERT_EQ(syncFenceHandle.getNativeHandle(), nullptr); ASSERT_EQ(fencedCallback, nullptr); executionStatus = result; + timing = {UINT64_MAX, UINT64_MAX}; } else if (syncFenceHandle.getNativeHandle()) { // If a sync fence is returned, try start another run waiting for the sync fence. ret = preparedModel->executeFenced(request, {syncFenceHandle}, From b1865b65938c43ce124852d2bfbd47e501a8ddc2 Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 19 Mar 2020 17:10:34 -0700 Subject: [PATCH 5/5] Remove extra tests from NNAPI VTS validation tests Do not run validation on "inputs_as_internal" and "all_tensors_as_inputs" variants. Bug: 138149072 Bug: 149840439 Test: mma Test: VtsHalNeuralnetworksV1_*TargetTest Change-Id: I0699ed6703e48b6c4bc0e7a392b79c12770f04c9 Merged-In: I0699ed6703e48b6c4bc0e7a392b79c12770f04c9 (cherry picked from commit 678a10600c6146bde78baa925e86dc99131c5d8a) --- neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- 12 files changed, 52 insertions(+), 4 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index 4ab228f85b..ae1e3a220d 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -160,6 +160,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h index f230a028f3..1a55c2f9c8 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp index 7f7dac056b..2c17796f2e 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp @@ -135,6 +135,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_0::vts::functional diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp index 14d300db71..a2338350aa 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp @@ -168,6 +168,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h index cf449ea42d..4b1a96e00d 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp index 04af6ec704..54e8802a54 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp @@ -138,6 +138,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_1::vts::functional diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp index aaaafc7b72..35275b4e61 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp @@ -390,6 +390,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h index dfc980c169..98295ff64a 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp index 5853fa49f6..a60ec4d1d2 100644 --- a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp @@ -162,7 +162,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_2(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel(); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 260fa3f326..4dbac1645a 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -916,6 +916,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h index 38d6486478..4f05c48166 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp index f7bd6241ad..df1e4535be 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp @@ -186,7 +186,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_3(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel();