From 187c59715c89b4988fcd73b8a9a4bdfe56269697 Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Wed, 7 Nov 2018 09:33:59 -0800 Subject: [PATCH] Add 1.2 NN HAL interface for dynamic output shape. Let notify_1_2() notify output shapes. Document unspecified dimensions and rank. Bug: 73506513 Bug: 77234888 Test: NeuralNetworksTest_static Test: VtsHalNeuralnetworksV1_xTargetTest with 1.2 sample driver Change-Id: I01108913212d9f4aa47daf2f293ea19259925865 --- .../1.0/vts/functional/Callbacks.cpp | 9 ++- neuralnetworks/1.0/vts/functional/Callbacks.h | 69 ++++++++++++++++--- .../vts/functional/GeneratedTestHarness.cpp | 37 +++++++--- neuralnetworks/1.2/Android.bp | 1 + neuralnetworks/1.2/IExecutionCallback.hal | 13 +++- neuralnetworks/1.2/IPreparedModel.hal | 12 +++- neuralnetworks/1.2/types.hal | 25 +++++-- .../1.2/vts/functional/ValidateRequest.cpp | 9 ++- 8 files changed, 141 insertions(+), 34 deletions(-) diff --git a/neuralnetworks/1.0/vts/functional/Callbacks.cpp b/neuralnetworks/1.0/vts/functional/Callbacks.cpp index a1c5a1adfc..03afcd0751 100644 --- a/neuralnetworks/1.0/vts/functional/Callbacks.cpp +++ b/neuralnetworks/1.0/vts/functional/Callbacks.cpp @@ -139,8 +139,10 @@ Return ExecutionCallback::notify(ErrorStatus errorStatus) { return Void(); } -Return ExecutionCallback::notify_1_2(ErrorStatus errorStatus) { +Return ExecutionCallback::notify_1_2(ErrorStatus errorStatus, + const hidl_vec& outputShapes) { mErrorStatus = errorStatus; + mOutputShapes = outputShapes; CallbackBase::notify(); return Void(); } @@ -150,6 +152,11 @@ ErrorStatus ExecutionCallback::getStatus() { return mErrorStatus; } +const std::vector& ExecutionCallback::getOutputShapes() { + wait(); + return mOutputShapes; +} + } // namespace implementation } // namespace V1_2 } // namespace neuralnetworks diff --git a/neuralnetworks/1.0/vts/functional/Callbacks.h b/neuralnetworks/1.0/vts/functional/Callbacks.h index e89980d4db..46f29a60e7 100644 --- a/neuralnetworks/1.0/vts/functional/Callbacks.h +++ b/neuralnetworks/1.0/vts/functional/Callbacks.h @@ -275,8 +275,9 @@ class ExecutionCallback : public CallbackBase, public IExecutionCallback { * Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must * be called exactly once on a given ExecutionCallback object. * - * @param status Error status returned from asynchronously preparing the - * model; will be: + * @param status Error status returned from launching the asynchronous task + * (if the launch fails) or from the asynchronous task itself + * (if the launch succeeds). Must be: * - NONE if the asynchronous execution was successful * - DEVICE_UNAVAILABLE if driver is offline or busy * - GENERAL_FAILURE if there is an unspecified error @@ -285,27 +286,73 @@ class ExecutionCallback : public CallbackBase, public IExecutionCallback { * - INVALID_ARGUMENT if the input request is invalid */ Return notify(ErrorStatus status) override; - Return notify_1_2(ErrorStatus status) override; + + /** + * Similar to IExecutionCallback::notify, but for V1_2::IPreparedModel to + * also notify output shapes along with error status. + * + * @param status Error status returned from launching the asynchronous task + * (if the launch fails) or from the asynchronous task itself + * (if the launch succeeds). Must be: + * - NONE if the asynchronous execution was successful + * - DEVICE_UNAVAILABLE if driver is offline or busy + * - GENERAL_FAILURE if the asynchronous task resulted in an + * unspecified error + * - OUTPUT_INSUFFICIENT_SIZE if at least one output + * operand buffer is not large enough to store the + * corresponding output + * - INVALID_ARGUMENT if one of the input arguments to + * prepareModel is invalid + * @param outputShapes A list of shape information of model output operands. + * The index into "outputShapes" corresponds to the index + * of the output operand in the Request outputs vector. + * outputShapes must be empty unless the status is either + * NONE or OUTPUT_INSUFFICIENT_SIZE. + */ + Return notify_1_2(ErrorStatus status, const hidl_vec& outputShapes) override; /** * Retrieves the error status returned from the asynchronous task launched - * by IPreparedModel::execute. If IPreparedModel::execute has not finished + * by either IPreparedModel::execute or IPreparedModel::execute_1_2. If + * IPreparedModel::execute or IPreparedModel::execute_1_2 has not finished * asynchronously executing, this call will block until the asynchronous task * notifies the object. * - * @return status Error status returned from asynchronously preparing the - * model; will be: + * @return status Error status returned from launching the asynchronous task + * (if the launch fails) or from the asynchronous task itself + * (if the launch succeeds). Must be: * - NONE if the asynchronous execution was successful * - DEVICE_UNAVAILABLE if driver is offline or busy - * - GENERAL_FAILURE if there is an unspecified error - * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is - * not large enough to store the resultant values - * - INVALID_ARGUMENT if the input request is invalid + * - GENERAL_FAILURE if the asynchronous task resulted in an + * unspecified error + * - OUTPUT_INSUFFICIENT_SIZE if at least one output + * operand buffer is not large enough to store the + * corresponding output + * - INVALID_ARGUMENT if one of the input arguments to + * prepareModel is invalid */ ErrorStatus getStatus(); - private: + /** + * Retrieves the output shapes returned from the asynchronous task launched + * by IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not finished + * asynchronously executing, this call will block until the asynchronous task + * notifies the object. + * + * If the asynchronous task was launched by IPreparedModel::execute, an empty vector + * will be returned. + * + * @return outputShapes A list of shape information of model output operands. + * The index into "outputShapes" corresponds to the index + * of the output operand in the Request outputs vector. + * outputShapes must be empty unless the status is either + * NONE or OUTPUT_INSUFFICIENT_SIZE. + */ + const std::vector& getOutputShapes(); + + private: ErrorStatus mErrorStatus; + std::vector mOutputShapes; }; diff --git a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp index 3b4eb217aa..b5a860771f 100644 --- a/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp +++ b/neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp @@ -89,13 +89,24 @@ static Return ExecutePreparedModel(sp& prepar sp& callback) { return preparedModel->execute_1_2(request, callback); } -static Return ExecutePreparedModel(sp&, const Request&) { +static Return ExecutePreparedModel(sp&, const Request&, + hidl_vec*) { ADD_FAILURE() << "asking for synchronous execution at V1_0"; return ErrorStatus::GENERAL_FAILURE; } static Return ExecutePreparedModel(sp& preparedModel, - const Request& request) { - return preparedModel->executeSynchronously(request); + const Request& request, + hidl_vec* outputShapes) { + ErrorStatus result; + Return ret = preparedModel->executeSynchronously( + request, [&result, &outputShapes](ErrorStatus error, const hidl_vec& shapes) { + result = error; + *outputShapes = shapes; + }); + if (!ret.isOk()) { + return ErrorStatus::GENERAL_FAILURE; + } + return result; } enum class Synchronously { NO, YES }; const float kDefaultAtol = 1e-5f; @@ -197,6 +208,8 @@ void EvaluatePreparedModel(sp& preparedModel, std::functioncommit(); outputMemory->commit(); + ErrorStatus executionStatus; + hidl_vec outputShapes; if (sync == Synchronously::NO) { SCOPED_TRACE("asynchronous"); @@ -211,18 +224,24 @@ void EvaluatePreparedModel(sp& preparedModel, std::functionwait(); - ErrorStatus executionReturnStatus = executionCallback->getStatus(); - EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus); + executionStatus = executionCallback->getStatus(); + outputShapes = executionCallback->getOutputShapes(); } else { SCOPED_TRACE("synchronous"); // execute - Return executionStatus = ExecutePreparedModel( - preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}); - ASSERT_TRUE(executionStatus.isOk()); - EXPECT_EQ(ErrorStatus::NONE, static_cast(executionStatus)); + Return executionReturnStatus = ExecutePreparedModel( + preparedModel, {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, + &outputShapes); + ASSERT_TRUE(executionReturnStatus.isOk()); + executionStatus = static_cast(executionReturnStatus); } + ASSERT_EQ(ErrorStatus::NONE, executionStatus); + // TODO(xusongw): Check if the returned output shapes match with expectation once the + // sample driver implementation of dynamic output shape is finished. + ASSERT_EQ(outputShapes.size(), 0); + // validate results outputMemory->read(); copy_back(&test, outputs_info, outputPtr); diff --git a/neuralnetworks/1.2/Android.bp b/neuralnetworks/1.2/Android.bp index 7d131044d4..d8762b093c 100644 --- a/neuralnetworks/1.2/Android.bp +++ b/neuralnetworks/1.2/Android.bp @@ -27,6 +27,7 @@ hidl_interface { "Operation", "OperationType", "OperationTypeRange", + "OutputShape", ], gen_java: false, } diff --git a/neuralnetworks/1.2/IExecutionCallback.hal b/neuralnetworks/1.2/IExecutionCallback.hal index 667e0d6823..47de1b60ec 100644 --- a/neuralnetworks/1.2/IExecutionCallback.hal +++ b/neuralnetworks/1.2/IExecutionCallback.hal @@ -18,6 +18,7 @@ package android.hardware.neuralnetworks@1.2; import @1.0::ErrorStatus; import @1.0::IExecutionCallback; +import OutputShape; /** * IExecutionCallback must be used to return the error status result from an @@ -39,10 +40,16 @@ interface IExecutionCallback extends @1.0::IExecutionCallback { * - DEVICE_UNAVAILABLE if driver is offline or busy * - GENERAL_FAILURE if the asynchronous task resulted in an * unspecified error - * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is - * not large enough to store the resultant values + * - OUTPUT_INSUFFICIENT_SIZE if at least one output + * operand buffer is not large enough to store the + * corresponding output * - INVALID_ARGUMENT if one of the input arguments to * prepareModel is invalid + * @param outputShapes A list of shape information of model output operands. + * The index into "outputShapes" corresponds with to index + * of the output operand in the Request outputs vector. + * outputShapes must be empty unless the status is either + * NONE or OUTPUT_INSUFFICIENT_SIZE. */ - oneway notify_1_2(ErrorStatus status); + oneway notify_1_2(ErrorStatus status, vec outputShapes); }; diff --git a/neuralnetworks/1.2/IPreparedModel.hal b/neuralnetworks/1.2/IPreparedModel.hal index 4e91c6754e..044ca289b9 100644 --- a/neuralnetworks/1.2/IPreparedModel.hal +++ b/neuralnetworks/1.2/IPreparedModel.hal @@ -100,11 +100,17 @@ interface IPreparedModel extends @1.0::IPreparedModel { * - NONE if execution is performed successfully * - DEVICE_UNAVAILABLE if driver is offline or busy * - GENERAL_FAILURE if there is an unspecified error - * - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is - * not large enough to store the resultant values + * - OUTPUT_INSUFFICIENT_SIZE if at least one output + * operand buffer is not large enough to store the + * corresponding output * - INVALID_ARGUMENT if one of the input arguments is * invalid + * @return outputShapes A list of shape information of model output operands. + * The index into "outputShapes" corresponds to the index + * of the output operand in the Request outputs vector. + * outputShapes must be empty unless the status is either + * NONE or OUTPUT_INSUFFICIENT_SIZE. */ executeSynchronously(Request request) - generates (ErrorStatus status); + generates (ErrorStatus status, vec outputShapes); }; diff --git a/neuralnetworks/1.2/types.hal b/neuralnetworks/1.2/types.hal index 40c07e73b3..4738cc3f1a 100644 --- a/neuralnetworks/1.2/types.hal +++ b/neuralnetworks/1.2/types.hal @@ -234,9 +234,6 @@ struct Operand { * * For a scalar operand, dimensions.size() must be 0. * - * For a tensor operand, dimensions.size() must be at least 1; - * however, any of the dimensions may be unspecified. - * * A tensor operand with all dimensions specified has "fully * specified" dimensions. Whenever possible (i.e., whenever the * dimensions are known at model construction time), a tensor @@ -255,17 +252,20 @@ struct Operand { * . The operand has lifetime CONSTANT_COPY or * CONSTANT_REFERENCE. * - * . The operand has lifetime MODEL_INPUT or MODEL_OUTPUT. Fully + * . The operand has lifetime MODEL_INPUT. Fully * specified dimensions must either be present in the * Operand or they must be provided in the corresponding * RequestArgument. - * EXCEPTION: If the input or output is optional and omitted + * EXCEPTION: If the input is optional and omitted * (by setting the hasNoValue field of the corresponding * RequestArgument to true) then it need not have fully * specified dimensions. * * A tensor operand with some number of unspecified dimensions is * represented by setting each unspecified dimension to 0. + * + * A tensor operand with unspecified rank is represented by providing + * an empty dimensions vector. */ vec dimensions; @@ -397,3 +397,18 @@ struct Model { */ bool relaxComputationFloat32toFloat16; }; + +/** + * Describes the shape information of an output operand after execution. + */ +struct OutputShape { + /** + * Dimensions of the operand. + */ + vec dimensions; + + /** + * Whether the provided buffer size is sufficient for the output. + */ + bool isSufficient; +}; diff --git a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp index d80fbcf689..1eaea4b9a6 100644 --- a/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp +++ b/neuralnetworks/1.2/vts/functional/ValidateRequest.cpp @@ -110,15 +110,20 @@ static void validate(const sp& preparedModel, const std::string& executionCallback->wait(); ErrorStatus executionReturnStatus = executionCallback->getStatus(); + const auto& outputShapes = executionCallback->getOutputShapes(); ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus); + ASSERT_EQ(outputShapes.size(), 0); } { SCOPED_TRACE(message + " [executeSynchronously]"); - Return executeStatus = preparedModel->executeSynchronously(request); + Return executeStatus = preparedModel->executeSynchronously( + request, [](ErrorStatus error, const hidl_vec& outputShapes) { + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, error); + EXPECT_EQ(outputShapes.size(), 0); + }); ASSERT_TRUE(executeStatus.isOk()); - ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast(executeStatus)); } }