diff --git a/neuralnetworks/1.0/vts/functional/Android.bp b/neuralnetworks/1.0/vts/functional/Android.bp index 18f35c1a16..ffba45c947 100644 --- a/neuralnetworks/1.0/vts/functional/Android.bp +++ b/neuralnetworks/1.0/vts/functional/Android.bp @@ -28,6 +28,7 @@ cc_library_static { "android.hardware.neuralnetworks@1.2", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", + "libgmock", "libhidlmemory", "libneuralnetworks_utils", ], @@ -55,6 +56,7 @@ cc_test { "android.hardware.neuralnetworks@1.2", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", + "libgmock", "libhidlmemory", "libneuralnetworks_utils", "VtsHalNeuralnetworksTest_utils", diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index b8046c79b2..cc199787e4 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -36,16 +36,17 @@ namespace neuralnetworks { namespace generated_tests { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; +using ::test_helper::compare; +using ::test_helper::expectMultinomialDistributionWithinTolerance; using ::test_helper::filter; +using ::test_helper::Float32Operands; using ::test_helper::for_all; using ::test_helper::for_each; -using ::test_helper::resize_accordingly; -using ::test_helper::MixedTyped; -using ::test_helper::MixedTypedExampleType; -using ::test_helper::Float32Operands; using ::test_helper::Int32Operands; +using ::test_helper::MixedTyped; +using ::test_helper::MixedTypedExample; using ::test_helper::Quant8Operands; -using ::test_helper::compare; +using ::test_helper::resize_accordingly; template void copy_back_(MixedTyped* dst, const std::vector& ra, char* src) { @@ -66,7 +67,7 @@ void copy_back(MixedTyped* dst, const std::vector& ra, char* sr // Top level driver for models and examples generated by test_generator.py // Test driver for those generated from ml/nn/runtime/test/spec void EvaluatePreparedModel(sp& preparedModel, std::function is_ignored, - const std::vector& examples, float fpAtol = 1e-5f, + const std::vector& examples, float fpAtol = 1e-5f, float fpRtol = 1e-5f) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; @@ -75,8 +76,8 @@ void EvaluatePreparedModel(sp& preparedModel, std::function inputs_info, outputs_info; uint32_t inputSize = 0, outputSize = 0; @@ -176,12 +177,15 @@ void EvaluatePreparedModel(sp& preparedModel, std::function 0) { + expectMultinomialDistributionWithinTolerance(test, example); + } } } void Execute(const sp& device, std::function create_model, - std::function is_ignored, - const std::vector& examples) { + std::function is_ignored, const std::vector& examples) { V1_0::Model model = create_model(); // see if service can handle model @@ -225,8 +229,7 @@ void Execute(const sp& device, std::function c } void Execute(const sp& device, std::function create_model, - std::function is_ignored, - const std::vector& examples) { + std::function is_ignored, const std::vector& examples) { V1_1::Model model = create_model(); // see if service can handle model @@ -277,8 +280,7 @@ void Execute(const sp& device, std::function c // TODO: Reduce code duplication. void Execute(const sp& device, std::function create_model, - std::function is_ignored, - const std::vector& examples) { + std::function is_ignored, const std::vector& examples) { V1_2::Model model = create_model(); // see if service can handle model diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp index d84479c992..ac1ae60d85 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTests.cpp @@ -31,9 +31,9 @@ namespace hardware { namespace neuralnetworks { namespace generated_tests { -using ::test_helper::MixedTypedExampleType; +using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&); } // namespace generated_tests namespace V1_0 { @@ -43,9 +43,7 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; using ::android::nn::allocateSharedMemory; - -// Mixed-typed examples -typedef test_helper::MixedTypedExampleType MixedTypedExample; +using ::test_helper::MixedTypedExample; // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.0/vts/functional/Models.h b/neuralnetworks/1.0/vts/functional/Models.h index 751ab32aea..268e67159f 100644 --- a/neuralnetworks/1.0/vts/functional/Models.h +++ b/neuralnetworks/1.0/vts/functional/Models.h @@ -30,7 +30,7 @@ namespace V1_0 { namespace vts { namespace functional { -using MixedTypedExample = test_helper::MixedTypedExampleType; +using MixedTypedExample = test_helper::MixedTypedExample; #define FOR_EACH_TEST_MODEL(FN) \ FN(add_broadcast_quant8) \ diff --git a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp index 09c1878593..1d3dee3213 100644 --- a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp @@ -36,9 +36,9 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; using ::android::hidl::memory::V1_0::IMemory; -using test_helper::MixedTyped; -using test_helper::MixedTypedExampleType; using test_helper::for_all; +using test_helper::MixedTyped; +using test_helper::MixedTypedExample; ///////////////////////// UTILITY FUNCTIONS ///////////////////////// @@ -151,15 +151,15 @@ static void removeOutputTest(const sp& preparedModel, const Requ ///////////////////////////// ENTRY POINT ////////////////////////////////// -std::vector createRequests(const std::vector& examples) { +std::vector createRequests(const std::vector& examples) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; std::vector requests; - for (auto& example : examples) { - const MixedTyped& inputs = example.first; - const MixedTyped& outputs = example.second; + for (const MixedTypedExample& example : examples) { + const MixedTyped& inputs = example.operands.first; + const MixedTyped& outputs = example.operands.second; std::vector inputs_info, outputs_info; uint32_t inputSize = 0, outputSize = 0; diff --git a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp b/neuralnetworks/1.0/vts/functional/ValidationTests.cpp index 98fc1c59f4..d3cbcff0a7 100644 --- a/neuralnetworks/1.0/vts/functional/ValidationTests.cpp +++ b/neuralnetworks/1.0/vts/functional/ValidationTests.cpp @@ -27,7 +27,7 @@ namespace vts { namespace functional { // forward declarations -std::vector createRequests(const std::vector& examples); +std::vector createRequests(const std::vector<::test_helper::MixedTypedExample>& examples); // generate validation tests #define VTS_CURRENT_TEST_CASE(TestName) \ diff --git a/neuralnetworks/1.1/vts/functional/Android.bp b/neuralnetworks/1.1/vts/functional/Android.bp index 52a804a8a3..a1c0f1fba8 100644 --- a/neuralnetworks/1.1/vts/functional/Android.bp +++ b/neuralnetworks/1.1/vts/functional/Android.bp @@ -31,6 +31,7 @@ cc_test { "android.hardware.neuralnetworks@1.2", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", + "libgmock", "libhidlmemory", "libneuralnetworks_utils", "VtsHalNeuralnetworksTest_utils", diff --git a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp index 95c2b1aca0..1f49904b4e 100644 --- a/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.1/vts/functional/GeneratedTests.cpp @@ -31,9 +31,9 @@ namespace hardware { namespace neuralnetworks { namespace generated_tests { -using ::test_helper::MixedTypedExampleType; +using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&); } // namespace generated_tests namespace V1_1 { @@ -43,9 +43,7 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; using ::android::nn::allocateSharedMemory; - -// Mixed-typed examples -typedef generated_tests::MixedTypedExampleType MixedTypedExample; +using ::test_helper::MixedTypedExample; // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.1/vts/functional/Models.h b/neuralnetworks/1.1/vts/functional/Models.h index cc0fac1265..62bc95e4e4 100644 --- a/neuralnetworks/1.1/vts/functional/Models.h +++ b/neuralnetworks/1.1/vts/functional/Models.h @@ -31,7 +31,7 @@ namespace V1_1 { namespace vts { namespace functional { -using MixedTypedExample = test_helper::MixedTypedExampleType; +using MixedTypedExample = test_helper::MixedTypedExample; #define FOR_EACH_TEST_MODEL(FN) \ FN(add) \ diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp index 687b760e76..e7d96c72c2 100644 --- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp @@ -36,9 +36,9 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; using ::android::hidl::memory::V1_0::IMemory; -using test_helper::MixedTyped; -using test_helper::MixedTypedExampleType; using test_helper::for_all; +using test_helper::MixedTyped; +using test_helper::MixedTypedExample; ///////////////////////// UTILITY FUNCTIONS ///////////////////////// @@ -152,15 +152,15 @@ static void removeOutputTest(const sp& preparedModel, const Requ ///////////////////////////// ENTRY POINT ////////////////////////////////// -std::vector createRequests(const std::vector& examples) { +std::vector createRequests(const std::vector& examples) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; std::vector requests; for (auto& example : examples) { - const MixedTyped& inputs = example.first; - const MixedTyped& outputs = example.second; + const MixedTyped& inputs = example.operands.first; + const MixedTyped& outputs = example.operands.second; std::vector inputs_info, outputs_info; uint32_t inputSize = 0, outputSize = 0; diff --git a/neuralnetworks/1.2/vts/functional/Android.bp b/neuralnetworks/1.2/vts/functional/Android.bp index 2dc19ccbe4..09d0dc3da6 100644 --- a/neuralnetworks/1.2/vts/functional/Android.bp +++ b/neuralnetworks/1.2/vts/functional/Android.bp @@ -31,6 +31,7 @@ cc_test { "android.hardware.neuralnetworks@1.2", "android.hidl.allocator@1.0", "android.hidl.memory@1.0", + "libgmock", "libhidlmemory", "libneuralnetworks_utils", "VtsHalNeuralnetworksTest_utils", diff --git a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp index 662c531340..e87fa6b6e1 100644 --- a/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp +++ b/neuralnetworks/1.2/vts/functional/GeneratedTests.cpp @@ -31,9 +31,9 @@ namespace hardware { namespace neuralnetworks { namespace generated_tests { -using ::test_helper::MixedTypedExampleType; +using ::test_helper::MixedTypedExample; extern void Execute(const sp&, std::function, - std::function, const std::vector&); + std::function, const std::vector&); } // namespace generated_tests namespace V1_2 { @@ -43,9 +43,7 @@ namespace functional { using ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; using ::android::nn::allocateSharedMemory; - -// Mixed-typed examples -typedef generated_tests::MixedTypedExampleType MixedTypedExample; +using ::test_helper::MixedTypedExample; // in frameworks/ml/nn/runtime/tests/generated/ #include "all_generated_V1_0_vts_tests.cpp" diff --git a/neuralnetworks/1.2/vts/functional/Models.h b/neuralnetworks/1.2/vts/functional/Models.h index f3769bc677..2d512feec2 100644 --- a/neuralnetworks/1.2/vts/functional/Models.h +++ b/neuralnetworks/1.2/vts/functional/Models.h @@ -32,7 +32,7 @@ namespace V1_2 { namespace vts { namespace functional { -using MixedTypedExample = test_helper::MixedTypedExampleType; +using MixedTypedExample = test_helper::MixedTypedExample; #define FOR_EACH_TEST_MODEL(FN) \ FN(add) \ @@ -243,6 +243,7 @@ using MixedTypedExample = test_helper::MixedTypedExampleType; FN(pad_float_1) \ FN(pad_float_1_relaxed) \ FN(pad_relaxed) \ + FN(random_multinomial) \ FN(relu1_float_1) \ FN(relu1_float_1_relaxed) \ FN(relu1_float_2) \ diff --git a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp index f4476fa07a..b663535088 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp @@ -38,7 +38,7 @@ using ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCa using ::android::hidl::memory::V1_0::IMemory; using test_helper::for_all; using test_helper::MixedTyped; -using test_helper::MixedTypedExampleType; +using test_helper::MixedTypedExample; ///////////////////////// UTILITY FUNCTIONS ///////////////////////// @@ -152,15 +152,15 @@ static void removeOutputTest(const sp& preparedModel, const Requ ///////////////////////////// ENTRY POINT ////////////////////////////////// -std::vector createRequests(const std::vector& examples) { +std::vector createRequests(const std::vector& examples) { const uint32_t INPUT = 0; const uint32_t OUTPUT = 1; std::vector requests; for (auto& example : examples) { - const MixedTyped& inputs = example.first; - const MixedTyped& outputs = example.second; + const MixedTyped& inputs = example.operands.first; + const MixedTyped& outputs = example.operands.second; std::vector inputs_info, outputs_info; uint32_t inputSize = 0, outputSize = 0;