From b1865b65938c43ce124852d2bfbd47e501a8ddc2 Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Thu, 19 Mar 2020 17:10:34 -0700 Subject: [PATCH] Remove extra tests from NNAPI VTS validation tests Do not run validation on "inputs_as_internal" and "all_tensors_as_inputs" variants. Bug: 138149072 Bug: 149840439 Test: mma Test: VtsHalNeuralnetworksV1_*TargetTest Change-Id: I0699ed6703e48b6c4bc0e7a392b79c12770f04c9 Merged-In: I0699ed6703e48b6c4bc0e7a392b79c12770f04c9 (cherry picked from commit 678a10600c6146bde78baa925e86dc99131c5d8a) --- neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp | 4 ++++ neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h | 3 +++ neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp | 7 ++++++- 12 files changed, 52 insertions(+), 4 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index 4ab228f85b..ae1e3a220d 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -160,6 +160,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h index f230a028f3..1a55c2f9c8 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp index 7f7dac056b..2c17796f2e 100644 --- a/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.0/vts/functional/VtsHalNeuralnetworks.cpp @@ -135,6 +135,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_0::vts::functional diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp index 14d300db71..a2338350aa 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.cpp @@ -168,6 +168,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h index cf449ea42d..4b1a96e00d 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.1/vts/functional/GeneratedTestHarness.h @@ -37,6 +37,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp index 04af6ec704..54e8802a54 100644 --- a/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.1/vts/functional/VtsHalNeuralnetworks.cpp @@ -138,6 +138,11 @@ TEST_P(ValidationTest, Test) { validateEverything(kDevice, model, request); } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); } // namespace android::hardware::neuralnetworks::V1_1::vts::functional diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp index aaaafc7b72..35275b4e61 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.cpp @@ -390,6 +390,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h index dfc980c169..98295ff64a 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.2/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp index 5853fa49f6..a60ec4d1d2 100644 --- a/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.2/vts/functional/VtsHalNeuralnetworks.cpp @@ -162,7 +162,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_2(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel(); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp index 260fa3f326..4dbac1645a 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp @@ -916,6 +916,10 @@ std::vector getNamedModels(const FilterFn& filter) { return TestModelManager::get().getTestModels(filter); } +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + std::string printGeneratedTest(const testing::TestParamInfo& info) { const auto& [namedDevice, namedModel] = info.param; return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); diff --git a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h index 38d6486478..4f05c48166 100644 --- a/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h +++ b/neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h @@ -41,6 +41,9 @@ class GeneratedTestBase : public testing::TestWithParam { using FilterFn = std::function; std::vector getNamedModels(const FilterFn& filter); +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + std::string printGeneratedTest(const testing::TestParamInfo& info); #define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ diff --git a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp index f7bd6241ad..df1e4535be 100644 --- a/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp +++ b/neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp @@ -186,7 +186,12 @@ TEST_P(ValidationTest, Test) { } } -INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; }); +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); sp getPreparedModel_1_3(const sp& callback) { sp preparedModelV1_0 = callback->getPreparedModel();