From da1a69288054dcbe9e91943d408a65b3c59c4a09 Mon Sep 17 00:00:00 2001 From: Michael Butler Date: Wed, 11 Mar 2020 18:45:45 -0700 Subject: [PATCH] NNAPI VTS: Add validation for Priority This CL also cleans up the validation framework code. Bug: 67828197 Test: mma Test: VtsHalNeuralnetworksV1_*TargetTest Change-Id: I84661fb2b8204148788d10425ca0ac986158b15f --- .../1.0/vts/functional/ValidateModel.cpp | 15 +- .../1.0/vts/functional/ValidateRequest.cpp | 10 +- .../1.1/vts/functional/ValidateModel.cpp | 118 +++++++----- .../1.1/vts/functional/ValidateRequest.cpp | 10 +- .../1.2/vts/functional/ValidateBurst.cpp | 14 +- .../1.2/vts/functional/ValidateModel.cpp | 118 +++++++----- .../1.2/vts/functional/ValidateRequest.cpp | 10 +- .../1.3/vts/functional/ValidateBurst.cpp | 14 +- .../1.3/vts/functional/ValidateModel.cpp | 179 +++++++++++------- .../1.3/vts/functional/ValidateRequest.cpp | 10 +- 10 files changed, 300 insertions(+), 198 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/ValidateModel.cpp b/neuralnetworks/1.0/vts/functional/ValidateModel.cpp index cc15263ae2..79d85943b1 100644 --- a/neuralnetworks/1.0/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.0/vts/functional/ValidateModel.cpp @@ -24,6 +24,8 @@ namespace android::hardware::neuralnetworks::V1_0::vts::functional { using implementation::PreparedModelCallback; +using PrepareModelMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static void validateGetSupportedOperations(const sp& device, const std::string& message, @@ -54,12 +56,13 @@ static void validatePrepareModel(const sp& device, const std::string& m } // Primary validation function. This function will take a valid model, apply a -// mutation to it to invalidate the model, then pass it to interface calls that -// use the model. Note that the model here is passed by value, and any mutation -// to the model does not leave this function. -static void validate(const sp& device, const std::string& message, Model model, - const std::function& mutation) { - mutation(&model); +// mutation to invalidate the model, then pass these to supportedOperations and +// prepareModel. +static void validate(const sp& device, const std::string& message, + const Model& originalModel, const PrepareModelMutation& mutate) { + Model model = originalModel; + mutate(&model); + validateGetSupportedOperations(device, message, model); validatePrepareModel(device, message, model); } diff --git a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp index 05eefd13d8..0baa85bc8e 100644 --- a/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.0/vts/functional/ValidateRequest.cpp @@ -24,15 +24,17 @@ namespace android::hardware::neuralnetworks::V1_0::vts::functional { using implementation::ExecutionCallback; +using ExecutionMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// // Primary validation function. This function will take a valid request, apply a // mutation to it to invalidate the request, then pass it to interface calls -// that use the request. Note that the request here is passed by value, and any -// mutation to the request does not leave this function. +// that use the request. static void validate(const sp& preparedModel, const std::string& message, - Request request, const std::function& mutation) { - mutation(&request); + const Request& originalRequest, const ExecutionMutation& mutate) { + Request request = originalRequest; + mutate(&request); SCOPED_TRACE(message + " [execute]"); sp executionCallback = new ExecutionCallback(); diff --git a/neuralnetworks/1.1/vts/functional/ValidateModel.cpp b/neuralnetworks/1.1/vts/functional/ValidateModel.cpp index 0629a1edf9..3b6f0f8300 100644 --- a/neuralnetworks/1.1/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.1/vts/functional/ValidateModel.cpp @@ -30,6 +30,8 @@ using V1_0::OperandLifeTime; using V1_0::OperandType; using V1_0::implementation::PreparedModelCallback; +using PrepareModelMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static void validateGetSupportedOperations(const sp& device, const std::string& message, @@ -67,16 +69,19 @@ static bool validExecutionPreference(ExecutionPreference preference) { } // Primary validation function. This function will take a valid model, apply a -// mutation to it to invalidate the model, then pass it to interface calls that -// use the model. Note that the model here is passed by value, and any mutation -// to the model does not leave this function. -static void validate(const sp& device, const std::string& message, Model model, - const std::function& mutation, - ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) { - mutation(&model); +// mutation to invalidate either the model or the execution preference, then +// pass these to supportedOperations and/or prepareModel if that method is +// called with an invalid argument. +static void validate(const sp& device, const std::string& message, + const Model& originalModel, const PrepareModelMutation& mutate) { + Model model = originalModel; + ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER; + mutate(&model, &preference); + if (validExecutionPreference(preference)) { validateGetSupportedOperations(device, message, model); } + validatePrepareModel(device, message, model, preference); } @@ -115,9 +120,11 @@ static void mutateOperandTypeTest(const sp& device, const Model& model) const std::string message = "mutateOperandTypeTest: operand " + std::to_string(operand) + " set to value " + std::to_string(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - model->operands[operand].type = static_cast(invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*) { + model->operands[operand].type = + static_cast(invalidOperandType); + }); } } } @@ -144,9 +151,10 @@ static void mutateOperandRankTest(const sp& device, const Model& model) const uint32_t invalidRank = getInvalidRank(model.operands[operand].type); const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + " has rank of " + std::to_string(invalidRank); - validate(device, message, model, [operand, invalidRank](Model* model) { - model->operands[operand].dimensions = std::vector(invalidRank, 0); - }); + validate(device, message, model, + [operand, invalidRank](Model* model, ExecutionPreference*) { + model->operands[operand].dimensions = std::vector(invalidRank, 0); + }); } } @@ -173,9 +181,10 @@ static void mutateOperandScaleTest(const sp& device, const Model& model const float invalidScale = getInvalidScale(model.operands[operand].type); const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) + " has scale of " + std::to_string(invalidScale); - validate(device, message, model, [operand, invalidScale](Model* model) { - model->operands[operand].scale = invalidScale; - }); + validate(device, message, model, + [operand, invalidScale](Model* model, ExecutionPreference*) { + model->operands[operand].scale = invalidScale; + }); } } @@ -204,9 +213,10 @@ static void mutateOperandZeroPointTest(const sp& device, const Model& m const std::string message = "mutateOperandZeroPointTest: operand " + std::to_string(operand) + " has zero point of " + std::to_string(invalidZeroPoint); - validate(device, message, model, [operand, invalidZeroPoint](Model* model) { - model->operands[operand].zeroPoint = invalidZeroPoint; - }); + validate(device, message, model, + [operand, invalidZeroPoint](Model* model, ExecutionPreference*) { + model->operands[operand].zeroPoint = invalidZeroPoint; + }); } } } @@ -282,9 +292,10 @@ static void mutateOperationOperandTypeTest(const sp& device, const Mode const std::string message = "mutateOperationOperandTypeTest: operand " + std::to_string(operand) + " set to type " + toString(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - mutateOperand(&model->operands[operand], invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*) { + mutateOperand(&model->operands[operand], invalidOperandType); + }); } } } @@ -304,10 +315,11 @@ static void mutateOperationTypeTest(const sp& device, const Model& mode const std::string message = "mutateOperationTypeTest: operation " + std::to_string(operation) + " set to value " + std::to_string(invalidOperationType); - validate(device, message, model, [operation, invalidOperationType](Model* model) { - model->operations[operation].type = - static_cast(invalidOperationType); - }); + validate(device, message, model, + [operation, invalidOperationType](Model* model, ExecutionPreference*) { + model->operations[operation].type = + static_cast(invalidOperationType); + }); } } } @@ -321,9 +333,10 @@ static void mutateOperationInputOperandIndexTest(const sp& device, cons const std::string message = "mutateOperationInputOperandIndexTest: operation " + std::to_string(operation) + " input " + std::to_string(input); - validate(device, message, model, [operation, input, invalidOperand](Model* model) { - model->operations[operation].inputs[input] = invalidOperand; - }); + validate(device, message, model, + [operation, input, invalidOperand](Model* model, ExecutionPreference*) { + model->operations[operation].inputs[input] = invalidOperand; + }); } } } @@ -337,9 +350,10 @@ static void mutateOperationOutputOperandIndexTest(const sp& device, con const std::string message = "mutateOperationOutputOperandIndexTest: operation " + std::to_string(operation) + " output " + std::to_string(output); - validate(device, message, model, [operation, output, invalidOperand](Model* model) { - model->operations[operation].outputs[output] = invalidOperand; - }); + validate(device, message, model, + [operation, output, invalidOperand](Model* model, ExecutionPreference*) { + model->operations[operation].outputs[output] = invalidOperand; + }); } } } @@ -372,7 +386,7 @@ static void removeOperandTest(const sp& device, const Model& model) { for (size_t operand = 0; operand < model.operands.size(); ++operand) { const std::string message = "removeOperandTest: operand " + std::to_string(operand); validate(device, message, model, - [operand](Model* model) { removeOperand(model, operand); }); + [operand](Model* model, ExecutionPreference*) { removeOperand(model, operand); }); } } @@ -388,8 +402,9 @@ static void removeOperation(Model* model, uint32_t index) { static void removeOperationTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.operations.size(); ++operation) { const std::string message = "removeOperationTest: operation " + std::to_string(operation); - validate(device, message, model, - [operation](Model* model) { removeOperation(model, operation); }); + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { + removeOperation(model, operation); + }); } } @@ -409,11 +424,12 @@ static void removeOperationInputTest(const sp& device, const Model& mod const std::string message = "removeOperationInputTest: operation " + std::to_string(operation) + ", input " + std::to_string(input); - validate(device, message, model, [operation, input](Model* model) { - uint32_t operand = model->operations[operation].inputs[input]; - model->operands[operand].numberOfConsumers--; - hidl_vec_removeAt(&model->operations[operation].inputs, input); - }); + validate(device, message, model, + [operation, input](Model* model, ExecutionPreference*) { + uint32_t operand = model->operations[operation].inputs[input]; + model->operands[operand].numberOfConsumers--; + hidl_vec_removeAt(&model->operations[operation].inputs, input); + }); } } } @@ -426,9 +442,10 @@ static void removeOperationOutputTest(const sp& device, const Model& mo const std::string message = "removeOperationOutputTest: operation " + std::to_string(operation) + ", output " + std::to_string(output); - validate(device, message, model, [operation, output](Model* model) { - hidl_vec_removeAt(&model->operations[operation].outputs, output); - }); + validate(device, message, model, + [operation, output](Model* model, ExecutionPreference*) { + hidl_vec_removeAt(&model->operations[operation].outputs, output); + }); } } } @@ -444,7 +461,7 @@ static void removeOperationOutputTest(const sp& device, const Model& mo static void addOperationInputTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.operations.size(); ++operation) { const std::string message = "addOperationInputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT); hidl_vec_push_back(&model->operations[operation].inputs, index); hidl_vec_push_back(&model->inputIndexes, index); @@ -458,7 +475,7 @@ static void addOperationOutputTest(const sp& device, const Model& model for (size_t operation = 0; operation < model.operations.size(); ++operation) { const std::string message = "addOperationOutputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT); hidl_vec_push_back(&model->operations[operation].outputs, index); hidl_vec_push_back(&model->outputIndexes, index); @@ -474,12 +491,13 @@ static const int32_t invalidExecutionPreferences[] = { }; static void mutateExecutionPreferenceTest(const sp& device, const Model& model) { - for (int32_t preference : invalidExecutionPreferences) { + for (int32_t invalidPreference : invalidExecutionPreferences) { const std::string message = - "mutateExecutionPreferenceTest: preference " + std::to_string(preference); - validate( - device, message, model, [](Model*) {}, - static_cast(preference)); + "mutateExecutionPreferenceTest: preference " + std::to_string(invalidPreference); + validate(device, message, model, + [invalidPreference](Model*, ExecutionPreference* preference) { + *preference = static_cast(invalidPreference); + }); } } diff --git a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp index 9684eb2b30..291433519e 100644 --- a/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.1/vts/functional/ValidateRequest.cpp @@ -28,15 +28,17 @@ using V1_0::IPreparedModel; using V1_0::Request; using V1_0::implementation::ExecutionCallback; +using ExecutionMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// // Primary validation function. This function will take a valid request, apply a // mutation to it to invalidate the request, then pass it to interface calls -// that use the request. Note that the request here is passed by value, and any -// mutation to the request does not leave this function. +// that use the request. static void validate(const sp& preparedModel, const std::string& message, - Request request, const std::function& mutation) { - mutation(&request); + const Request& originalRequest, const ExecutionMutation& mutate) { + Request request = originalRequest; + mutate(&request); SCOPED_TRACE(message + " [execute]"); sp executionCallback = new ExecutionCallback(); diff --git a/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp b/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp index ec9629bccb..9a22490383 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateBurst.cpp @@ -38,6 +38,8 @@ using V1_0::ErrorStatus; using V1_0::Request; using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback; +using BurstExecutionMutation = std::function*)>; + // This constant value represents the length of an FMQ that is large enough to // return a result from a burst execution for all of the generated test cases. constexpr size_t kExecutionBurstChannelLength = 1024; @@ -116,13 +118,13 @@ static void createBurstWithResultChannelLength( // Primary validation function. This function will take a valid serialized // request, apply a mutation to it to invalidate the serialized request, then -// pass it to interface calls that use the serialized request. Note that the -// serialized request here is passed by value, and any mutation to the -// serialized request does not leave this function. +// pass it to interface calls that use the serialized request. static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::string& message, std::vector serialized, - const std::function*)>& mutation) { - mutation(&serialized); + const std::string& message, + const std::vector& originalSerialized, + const BurstExecutionMutation& mutate) { + std::vector serialized = originalSerialized; + mutate(&serialized); // skip if packet is too large to send if (serialized.size() > kExecutionBurstChannelLength) { diff --git a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp index 30530beacc..7451f095bf 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateModel.cpp @@ -29,6 +29,8 @@ using V1_0::OperandLifeTime; using V1_1::ExecutionPreference; using HidlToken = hidl_array(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; +using PrepareModelMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static void validateGetSupportedOperations(const sp& device, const std::string& message, @@ -67,16 +69,19 @@ static bool validExecutionPreference(ExecutionPreference preference) { } // Primary validation function. This function will take a valid model, apply a -// mutation to it to invalidate the model, then pass it to interface calls that -// use the model. Note that the model here is passed by value, and any mutation -// to the model does not leave this function. -static void validate(const sp& device, const std::string& message, Model model, - const std::function& mutation, - ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) { - mutation(&model); +// mutation to invalidate either the model or the execution preference, then +// pass these to supportedOperations and/or prepareModel if that method is +// called with an invalid argument. +static void validate(const sp& device, const std::string& message, + const Model& originalModel, const PrepareModelMutation& mutate) { + Model model = originalModel; + ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER; + mutate(&model, &preference); + if (validExecutionPreference(preference)) { validateGetSupportedOperations(device, message, model); } + validatePrepareModel(device, message, model, preference); } @@ -115,9 +120,11 @@ static void mutateOperandTypeTest(const sp& device, const Model& model) const std::string message = "mutateOperandTypeTest: operand " + std::to_string(operand) + " set to value " + std::to_string(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - model->operands[operand].type = static_cast(invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*) { + model->operands[operand].type = + static_cast(invalidOperandType); + }); } } } @@ -155,9 +162,10 @@ static void mutateOperandRankTest(const sp& device, const Model& model) } const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + " has rank of " + std::to_string(invalidRank); - validate(device, message, model, [operand, invalidRank](Model* model) { - model->operands[operand].dimensions = std::vector(invalidRank, 0); - }); + validate(device, message, model, + [operand, invalidRank](Model* model, ExecutionPreference*) { + model->operands[operand].dimensions = std::vector(invalidRank, 0); + }); } } @@ -192,9 +200,10 @@ static void mutateOperandScaleTest(const sp& device, const Model& model const float invalidScale = getInvalidScale(model.operands[operand].type); const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) + " has scale of " + std::to_string(invalidScale); - validate(device, message, model, [operand, invalidScale](Model* model) { - model->operands[operand].scale = invalidScale; - }); + validate(device, message, model, + [operand, invalidScale](Model* model, ExecutionPreference*) { + model->operands[operand].scale = invalidScale; + }); } } @@ -234,9 +243,10 @@ static void mutateOperandZeroPointTest(const sp& device, const Model& m const std::string message = "mutateOperandZeroPointTest: operand " + std::to_string(operand) + " has zero point of " + std::to_string(invalidZeroPoint); - validate(device, message, model, [operand, invalidZeroPoint](Model* model) { - model->operands[operand].zeroPoint = invalidZeroPoint; - }); + validate(device, message, model, + [operand, invalidZeroPoint](Model* model, ExecutionPreference*) { + model->operands[operand].zeroPoint = invalidZeroPoint; + }); } } } @@ -386,9 +396,10 @@ static void mutateOperationOperandTypeTest(const sp& device, const Mode const std::string message = "mutateOperationOperandTypeTest: operand " + std::to_string(operand) + " set to type " + toString(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - mutateOperand(&model->operands[operand], invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*) { + mutateOperand(&model->operands[operand], invalidOperandType); + }); } } } @@ -407,10 +418,11 @@ static void mutateOperationTypeTest(const sp& device, const Model& mode const std::string message = "mutateOperationTypeTest: operation " + std::to_string(operation) + " set to value " + std::to_string(invalidOperationType); - validate(device, message, model, [operation, invalidOperationType](Model* model) { - model->operations[operation].type = - static_cast(invalidOperationType); - }); + validate(device, message, model, + [operation, invalidOperationType](Model* model, ExecutionPreference*) { + model->operations[operation].type = + static_cast(invalidOperationType); + }); } } } @@ -424,9 +436,10 @@ static void mutateOperationInputOperandIndexTest(const sp& device, cons const std::string message = "mutateOperationInputOperandIndexTest: operation " + std::to_string(operation) + " input " + std::to_string(input); - validate(device, message, model, [operation, input, invalidOperand](Model* model) { - model->operations[operation].inputs[input] = invalidOperand; - }); + validate(device, message, model, + [operation, input, invalidOperand](Model* model, ExecutionPreference*) { + model->operations[operation].inputs[input] = invalidOperand; + }); } } } @@ -440,9 +453,10 @@ static void mutateOperationOutputOperandIndexTest(const sp& device, con const std::string message = "mutateOperationOutputOperandIndexTest: operation " + std::to_string(operation) + " output " + std::to_string(output); - validate(device, message, model, [operation, output, invalidOperand](Model* model) { - model->operations[operation].outputs[output] = invalidOperand; - }); + validate(device, message, model, + [operation, output, invalidOperand](Model* model, ExecutionPreference*) { + model->operations[operation].outputs[output] = invalidOperand; + }); } } } @@ -503,7 +517,7 @@ static void removeOperandTest(const sp& device, const Model& model) { } const std::string message = "removeOperandTest: operand " + std::to_string(operand); validate(device, message, model, - [operand](Model* model) { removeOperand(model, operand); }); + [operand](Model* model, ExecutionPreference*) { removeOperand(model, operand); }); } } @@ -519,8 +533,9 @@ static void removeOperation(Model* model, uint32_t index) { static void removeOperationTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.operations.size(); ++operation) { const std::string message = "removeOperationTest: operation " + std::to_string(operation); - validate(device, message, model, - [operation](Model* model) { removeOperation(model, operation); }); + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { + removeOperation(model, operation); + }); } } @@ -601,11 +616,12 @@ static void removeOperationInputTest(const sp& device, const Model& mod const std::string message = "removeOperationInputTest: operation " + std::to_string(operation) + ", input " + std::to_string(input); - validate(device, message, model, [operation, input](Model* model) { - uint32_t operand = model->operations[operation].inputs[input]; - model->operands[operand].numberOfConsumers--; - hidl_vec_removeAt(&model->operations[operation].inputs, input); - }); + validate(device, message, model, + [operation, input](Model* model, ExecutionPreference*) { + uint32_t operand = model->operations[operation].inputs[input]; + model->operands[operand].numberOfConsumers--; + hidl_vec_removeAt(&model->operations[operation].inputs, input); + }); } } } @@ -618,9 +634,10 @@ static void removeOperationOutputTest(const sp& device, const Model& mo const std::string message = "removeOperationOutputTest: operation " + std::to_string(operation) + ", output " + std::to_string(output); - validate(device, message, model, [operation, output](Model* model) { - hidl_vec_removeAt(&model->operations[operation].outputs, output); - }); + validate(device, message, model, + [operation, output](Model* model, ExecutionPreference*) { + hidl_vec_removeAt(&model->operations[operation].outputs, output); + }); } } } @@ -651,7 +668,7 @@ static void addOperationInputTest(const sp& device, const Model& model) continue; } const std::string message = "addOperationInputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT); hidl_vec_push_back(&model->operations[operation].inputs, index); hidl_vec_push_back(&model->inputIndexes, index); @@ -665,7 +682,7 @@ static void addOperationOutputTest(const sp& device, const Model& model for (size_t operation = 0; operation < model.operations.size(); ++operation) { const std::string message = "addOperationOutputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { + validate(device, message, model, [operation](Model* model, ExecutionPreference*) { uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT); hidl_vec_push_back(&model->operations[operation].outputs, index); hidl_vec_push_back(&model->outputIndexes, index); @@ -681,12 +698,13 @@ static const int32_t invalidExecutionPreferences[] = { }; static void mutateExecutionPreferenceTest(const sp& device, const Model& model) { - for (int32_t preference : invalidExecutionPreferences) { + for (int32_t invalidPreference : invalidExecutionPreferences) { const std::string message = - "mutateExecutionPreferenceTest: preference " + std::to_string(preference); - validate( - device, message, model, [](Model*) {}, - static_cast(preference)); + "mutateExecutionPreferenceTest: preference " + std::to_string(invalidPreference); + validate(device, message, model, + [invalidPreference](Model*, ExecutionPreference* preference) { + *preference = static_cast(invalidPreference); + }); } } diff --git a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp index 7b5ff9b8e4..7a23c13fe0 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp @@ -31,6 +31,8 @@ using implementation::ExecutionCallback; using V1_0::ErrorStatus; using V1_0::Request; +using ExecutionMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static bool badTiming(Timing timing) { @@ -39,11 +41,11 @@ static bool badTiming(Timing timing) { // Primary validation function. This function will take a valid request, apply a // mutation to it to invalidate the request, then pass it to interface calls -// that use the request. Note that the request here is passed by value, and any -// mutation to the request does not leave this function. +// that use the request. static void validate(const sp& preparedModel, const std::string& message, - Request request, const std::function& mutation) { - mutation(&request); + const Request& originalRequest, const ExecutionMutation& mutate) { + Request request = originalRequest; + mutate(&request); // We'd like to test both with timing requested and without timing // requested. Rather than running each test both ways, we'll decide whether diff --git a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp b/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp index 6ff9dfd3a8..6ee7b1c534 100644 --- a/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp +++ b/neuralnetworks/1.3/vts/functional/ValidateBurst.cpp @@ -43,6 +43,8 @@ using V1_2::MeasureTiming; using V1_2::Timing; using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback; +using BurstExecutionMutation = std::function*)>; + // This constant value represents the length of an FMQ that is large enough to // return a result from a burst execution for all of the generated test cases. constexpr size_t kExecutionBurstChannelLength = 1024; @@ -122,13 +124,13 @@ static void createBurstWithResultChannelLength( // Primary validation function. This function will take a valid serialized // request, apply a mutation to it to invalidate the serialized request, then -// pass it to interface calls that use the serialized request. Note that the -// serialized request here is passed by value, and any mutation to the -// serialized request does not leave this function. +// pass it to interface calls that use the serialized request. static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver, - const std::string& message, std::vector serialized, - const std::function*)>& mutation) { - mutation(&serialized); + const std::string& message, + const std::vector& originalSerialized, + const BurstExecutionMutation& mutate) { + std::vector serialized = originalSerialized; + mutate(&serialized); // skip if packet is too large to send if (serialized.size() > kExecutionBurstChannelLength) { diff --git a/neuralnetworks/1.3/vts/functional/ValidateModel.cpp b/neuralnetworks/1.3/vts/functional/ValidateModel.cpp index 7da2da980b..4c0100e219 100644 --- a/neuralnetworks/1.3/vts/functional/ValidateModel.cpp +++ b/neuralnetworks/1.3/vts/functional/ValidateModel.cpp @@ -30,6 +30,8 @@ using V1_2::SymmPerChannelQuantParams; using HidlToken = hidl_array(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>; +using PrepareModelMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static void validateGetSupportedOperations(const sp& device, const std::string& message, @@ -44,13 +46,14 @@ static void validateGetSupportedOperations(const sp& device, const std: } static void validatePrepareModel(const sp& device, const std::string& message, - const Model& model, ExecutionPreference preference) { + const Model& model, ExecutionPreference preference, + Priority priority) { SCOPED_TRACE(message + " [prepareModel_1_3]"); sp preparedModelCallback = new PreparedModelCallback(); - Return prepareLaunchStatus = device->prepareModel_1_3( - model, preference, kDefaultPriority, {}, hidl_vec(), - hidl_vec(), HidlToken(), preparedModelCallback); + Return prepareLaunchStatus = + device->prepareModel_1_3(model, preference, priority, {}, hidl_vec(), + hidl_vec(), HidlToken(), preparedModelCallback); ASSERT_TRUE(prepareLaunchStatus.isOk()); ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(prepareLaunchStatus)); @@ -67,18 +70,26 @@ static bool validExecutionPreference(ExecutionPreference preference) { preference == ExecutionPreference::SUSTAINED_SPEED; } +static bool validExecutionPriority(Priority priority) { + return priority == Priority::LOW || priority == Priority::MEDIUM || priority == Priority::HIGH; +} + // Primary validation function. This function will take a valid model, apply a -// mutation to it to invalidate the model, then pass it to interface calls that -// use the model. Note that the model here is passed by value, and any mutation -// to the model does not leave this function. -static void validate(const sp& device, const std::string& message, Model model, - const std::function& mutation, - ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) { - mutation(&model); - if (validExecutionPreference(preference)) { +// mutation to invalidate the model, the execution preference, or the priority, +// then pass these to supportedOperations and/or prepareModel if that method is +// called with an invalid argument. +static void validate(const sp& device, const std::string& message, + const Model& originalModel, const PrepareModelMutation& mutate) { + Model model = originalModel; + ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER; + Priority priority = kDefaultPriority; + mutate(&model, &preference, &priority); + + if (validExecutionPreference(preference) && validExecutionPriority(priority)) { validateGetSupportedOperations(device, message, model); } - validatePrepareModel(device, message, model, preference); + + validatePrepareModel(device, message, model, preference, priority); } static uint32_t addOperand(Model* model) { @@ -116,9 +127,11 @@ static void mutateOperandTypeTest(const sp& device, const Model& model) const std::string message = "mutateOperandTypeTest: operand " + std::to_string(operand) + " set to value " + std::to_string(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - model->main.operands[operand].type = static_cast(invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].type = + static_cast(invalidOperandType); + }); } } } @@ -156,9 +169,11 @@ static void mutateOperandRankTest(const sp& device, const Model& model) } const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + " has rank of " + std::to_string(invalidRank); - validate(device, message, model, [operand, invalidRank](Model* model) { - model->main.operands[operand].dimensions = std::vector(invalidRank, 0); - }); + validate(device, message, model, + [operand, invalidRank](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].dimensions = + std::vector(invalidRank, 0); + }); } } @@ -194,9 +209,10 @@ static void mutateOperandScaleTest(const sp& device, const Model& model const float invalidScale = getInvalidScale(model.main.operands[operand].type); const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) + " has scale of " + std::to_string(invalidScale); - validate(device, message, model, [operand, invalidScale](Model* model) { - model->main.operands[operand].scale = invalidScale; - }); + validate(device, message, model, + [operand, invalidScale](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].scale = invalidScale; + }); } } @@ -237,9 +253,10 @@ static void mutateOperandZeroPointTest(const sp& device, const Model& m const std::string message = "mutateOperandZeroPointTest: operand " + std::to_string(operand) + " has zero point of " + std::to_string(invalidZeroPoint); - validate(device, message, model, [operand, invalidZeroPoint](Model* model) { - model->main.operands[operand].zeroPoint = invalidZeroPoint; - }); + validate(device, message, model, + [operand, invalidZeroPoint](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].zeroPoint = invalidZeroPoint; + }); } } } @@ -425,9 +442,10 @@ static void mutateOperationOperandTypeTest(const sp& device, const Mode const std::string message = "mutateOperationOperandTypeTest: operand " + std::to_string(operand) + " set to type " + toString(invalidOperandType); - validate(device, message, model, [operand, invalidOperandType](Model* model) { - mutateOperand(&model->main.operands[operand], invalidOperandType); - }); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*, Priority*) { + mutateOperand(&model->main.operands[operand], invalidOperandType); + }); } } } @@ -446,10 +464,12 @@ static void mutateOperationTypeTest(const sp& device, const Model& mode const std::string message = "mutateOperationTypeTest: operation " + std::to_string(operation) + " set to value " + std::to_string(invalidOperationType); - validate(device, message, model, [operation, invalidOperationType](Model* model) { - model->main.operations[operation].type = - static_cast(invalidOperationType); - }); + validate(device, message, model, + [operation, invalidOperationType](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].type = + static_cast(invalidOperationType); + }); } } } @@ -463,9 +483,11 @@ static void mutateOperationInputOperandIndexTest(const sp& device, cons const std::string message = "mutateOperationInputOperandIndexTest: operation " + std::to_string(operation) + " input " + std::to_string(input); - validate(device, message, model, [operation, input, invalidOperand](Model* model) { - model->main.operations[operation].inputs[input] = invalidOperand; - }); + validate(device, message, model, + [operation, input, invalidOperand](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].inputs[input] = invalidOperand; + }); } } } @@ -480,9 +502,11 @@ static void mutateOperationOutputOperandIndexTest(const sp& device, con const std::string message = "mutateOperationOutputOperandIndexTest: operation " + std::to_string(operation) + " output " + std::to_string(output); - validate(device, message, model, [operation, output, invalidOperand](Model* model) { - model->main.operations[operation].outputs[output] = invalidOperand; - }); + validate(device, message, model, + [operation, output, invalidOperand](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].outputs[output] = invalidOperand; + }); } } } @@ -548,8 +572,9 @@ static void removeOperandTest(const sp& device, const Model& model) { continue; } const std::string message = "removeOperandTest: operand " + std::to_string(operand); - validate(device, message, model, - [operand](Model* model) { removeOperand(model, operand); }); + validate(device, message, model, [operand](Model* model, ExecutionPreference*, Priority*) { + removeOperand(model, operand); + }); } } @@ -566,7 +591,9 @@ static void removeOperationTest(const sp& device, const Model& model) { for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { const std::string message = "removeOperationTest: operation " + std::to_string(operation); validate(device, message, model, - [operation](Model* model) { removeOperation(model, operation); }); + [operation](Model* model, ExecutionPreference*, Priority*) { + removeOperation(model, operation); + }); } } @@ -654,11 +681,12 @@ static void removeOperationInputTest(const sp& device, const Model& mod const std::string message = "removeOperationInputTest: operation " + std::to_string(operation) + ", input " + std::to_string(input); - validate(device, message, model, [operation, input](Model* model) { - uint32_t operand = model->main.operations[operation].inputs[input]; - model->main.operands[operand].numberOfConsumers--; - hidl_vec_removeAt(&model->main.operations[operation].inputs, input); - }); + validate(device, message, model, + [operation, input](Model* model, ExecutionPreference*, Priority*) { + uint32_t operand = model->main.operations[operation].inputs[input]; + model->main.operands[operand].numberOfConsumers--; + hidl_vec_removeAt(&model->main.operations[operation].inputs, input); + }); } } } @@ -672,9 +700,10 @@ static void removeOperationOutputTest(const sp& device, const Model& mo const std::string message = "removeOperationOutputTest: operation " + std::to_string(operation) + ", output " + std::to_string(output); - validate(device, message, model, [operation, output](Model* model) { - hidl_vec_removeAt(&model->main.operations[operation].outputs, output); - }); + validate(device, message, model, + [operation, output](Model* model, ExecutionPreference*, Priority*) { + hidl_vec_removeAt(&model->main.operations[operation].outputs, output); + }); } } } @@ -707,11 +736,12 @@ static void addOperationInputTest(const sp& device, const Model& model) continue; } const std::string message = "addOperationInputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { - uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_INPUT); - hidl_vec_push_back(&model->main.operations[operation].inputs, index); - hidl_vec_push_back(&model->main.inputIndexes, index); - }); + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_INPUT); + hidl_vec_push_back(&model->main.operations[operation].inputs, index); + hidl_vec_push_back(&model->main.inputIndexes, index); + }); } } @@ -721,11 +751,12 @@ static void addOperationOutputTest(const sp& device, const Model& model for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { const std::string message = "addOperationOutputTest: operation " + std::to_string(operation); - validate(device, message, model, [operation](Model* model) { - uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_OUTPUT); - hidl_vec_push_back(&model->main.operations[operation].outputs, index); - hidl_vec_push_back(&model->main.outputIndexes, index); - }); + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_OUTPUT); + hidl_vec_push_back(&model->main.operations[operation].outputs, index); + hidl_vec_push_back(&model->main.outputIndexes, index); + }); } } @@ -737,12 +768,31 @@ static const int32_t invalidExecutionPreferences[] = { }; static void mutateExecutionPreferenceTest(const sp& device, const Model& model) { - for (int32_t preference : invalidExecutionPreferences) { + for (int32_t invalidPreference : invalidExecutionPreferences) { const std::string message = - "mutateExecutionPreferenceTest: preference " + std::to_string(preference); - validate( - device, message, model, [](Model*) {}, - static_cast(preference)); + "mutateExecutionPreferenceTest: preference " + std::to_string(invalidPreference); + validate(device, message, model, + [invalidPreference](Model*, ExecutionPreference* preference, Priority*) { + *preference = static_cast(invalidPreference); + }); + } +} + +///////////////////////// VALIDATE PRIORITY ///////////////////////// + +static const int32_t invalidPriorities[] = { + static_cast(Priority::LOW) - 1, // lower bound + static_cast(Priority::HIGH) + 1, // upper bound +}; + +static void mutateExecutionPriorityTest(const sp& device, const Model& model) { + for (int32_t invalidPriority : invalidPriorities) { + const std::string message = + "mutatePriorityTest: priority " + std::to_string(invalidPriority); + validate(device, message, model, + [invalidPriority](Model*, ExecutionPreference*, Priority* priority) { + *priority = static_cast(invalidPriority); + }); } } @@ -764,6 +814,7 @@ void validateModel(const sp& device, const Model& model) { addOperationInputTest(device, model); addOperationOutputTest(device, model); mutateExecutionPreferenceTest(device, model); + mutateExecutionPriorityTest(device, model); } } // namespace android::hardware::neuralnetworks::V1_3::vts::functional diff --git a/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp index 5e806e5c9b..1ae8b3f21d 100644 --- a/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.3/vts/functional/ValidateRequest.cpp @@ -34,6 +34,8 @@ using V1_2::MeasureTiming; using V1_2::OutputShape; using V1_2::Timing; +using ExecutionMutation = std::function; + ///////////////////////// UTILITY FUNCTIONS ///////////////////////// static bool badTiming(Timing timing) { @@ -42,11 +44,11 @@ static bool badTiming(Timing timing) { // Primary validation function. This function will take a valid request, apply a // mutation to it to invalidate the request, then pass it to interface calls -// that use the request. Note that the request here is passed by value, and any -// mutation to the request does not leave this function. +// that use the request. static void validate(const sp& preparedModel, const std::string& message, - Request request, const std::function& mutation) { - mutation(&request); + const Request& originalRequest, const ExecutionMutation& mutate) { + Request request = originalRequest; + mutate(&request); // We'd like to test both with timing requested and without timing // requested. Rather than running each test both ways, we'll decide whether