From c185e88ccf62ee47fd13d52f447b4db471701fce Mon Sep 17 00:00:00 2001 From: Lev Proleev Date: Tue, 15 Dec 2020 19:25:32 +0000 Subject: [PATCH] Implement VTS tests for NNAPI AIDL interface The tests are copied from HIDL 1.0-3 VTS tests and updated to use AIDL. Bug: 172922059 Test: VtsHalNeuralnetworksTargetTest Change-Id: Ife08409e9b46420685a1ccb0b3256286c973dbf5 Merged-In: Ife08409e9b46420685a1ccb0b3256286c973dbf5 (cherry picked from commit b38bb4f12a1ceb33ebd0dd798650a74a8ef9d20e) --- neuralnetworks/1.3/vts/functional/Android.bp | 1 + .../aidl/utils/include/nnapi/hal/aidl/Utils.h | 5 +- neuralnetworks/aidl/utils/src/Utils.cpp | 85 +- neuralnetworks/aidl/vts/OWNERS | 12 + neuralnetworks/aidl/vts/functional/Android.bp | 68 + .../aidl/vts/functional/AndroidTest.xml | 33 + .../aidl/vts/functional/BasicTests.cpp | 193 +++ .../aidl/vts/functional/Callbacks.cpp | 59 + .../aidl/vts/functional/Callbacks.h | 131 ++ .../functional/CompilationCachingTests.cpp | 1177 +++++++++++++++ .../vts/functional/GeneratedTestHarness.cpp | 925 ++++++++++++ .../vts/functional/GeneratedTestHarness.h | 88 ++ .../aidl/vts/functional/LogTestCaseToLogcat.h | 40 + .../aidl/vts/functional/MemoryDomainTests.cpp | 1176 +++++++++++++++ .../vts/functional/QualityOfServiceTests.cpp | 270 ++++ .../aidl/vts/functional/TestAssertions.cpp | 153 ++ .../aidl/vts/functional/TestMain.cpp | 27 + neuralnetworks/aidl/vts/functional/Utils.cpp | 252 ++++ neuralnetworks/aidl/vts/functional/Utils.h | 153 ++ .../aidl/vts/functional/ValidateModel.cpp | 1338 +++++++++++++++++ .../aidl/vts/functional/ValidateRequest.cpp | 126 ++ .../vts/functional/VtsHalNeuralnetworks.cpp | 194 +++ .../vts/functional/VtsHalNeuralnetworks.h | 61 + 23 files changed, 6543 insertions(+), 24 deletions(-) create mode 100644 neuralnetworks/aidl/vts/OWNERS create mode 100644 neuralnetworks/aidl/vts/functional/Android.bp create mode 100644 neuralnetworks/aidl/vts/functional/AndroidTest.xml create mode 100644 neuralnetworks/aidl/vts/functional/BasicTests.cpp create mode 100644 neuralnetworks/aidl/vts/functional/Callbacks.cpp create mode 100644 neuralnetworks/aidl/vts/functional/Callbacks.h create mode 100644 neuralnetworks/aidl/vts/functional/CompilationCachingTests.cpp create mode 100644 neuralnetworks/aidl/vts/functional/GeneratedTestHarness.cpp create mode 100644 neuralnetworks/aidl/vts/functional/GeneratedTestHarness.h create mode 100644 neuralnetworks/aidl/vts/functional/LogTestCaseToLogcat.h create mode 100644 neuralnetworks/aidl/vts/functional/MemoryDomainTests.cpp create mode 100644 neuralnetworks/aidl/vts/functional/QualityOfServiceTests.cpp create mode 100644 neuralnetworks/aidl/vts/functional/TestAssertions.cpp create mode 100644 neuralnetworks/aidl/vts/functional/TestMain.cpp create mode 100644 neuralnetworks/aidl/vts/functional/Utils.cpp create mode 100644 neuralnetworks/aidl/vts/functional/Utils.h create mode 100644 neuralnetworks/aidl/vts/functional/ValidateModel.cpp create mode 100644 neuralnetworks/aidl/vts/functional/ValidateRequest.cpp create mode 100644 neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.cpp create mode 100644 neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.h diff --git a/neuralnetworks/1.3/vts/functional/Android.bp b/neuralnetworks/1.3/vts/functional/Android.bp index b17d44559b..ee753bb951 100644 --- a/neuralnetworks/1.3/vts/functional/Android.bp +++ b/neuralnetworks/1.3/vts/functional/Android.bp @@ -57,6 +57,7 @@ cc_test { "VtsHalNeuralNetworksV1_0_utils", "VtsHalNeuralNetworksV1_2_utils", "VtsHalNeuralNetworksV1_3_utils", + "android.hardware.neuralnetworks-V1-ndk_platform", "android.hardware.neuralnetworks@1.0", "android.hardware.neuralnetworks@1.1", "android.hardware.neuralnetworks@1.2", diff --git a/neuralnetworks/aidl/utils/include/nnapi/hal/aidl/Utils.h b/neuralnetworks/aidl/utils/include/nnapi/hal/aidl/Utils.h index 802e70304a..79b511dc56 100644 --- a/neuralnetworks/aidl/utils/include/nnapi/hal/aidl/Utils.h +++ b/neuralnetworks/aidl/utils/include/nnapi/hal/aidl/Utils.h @@ -47,7 +47,10 @@ bool valid(const Type& halObject) { return result.has_value(); } -nn::GeneralResult copyModel(const Model& model); +nn::GeneralResult clone(const Memory& memory); +nn::GeneralResult clone(const Request& request); +nn::GeneralResult clone(const RequestMemoryPool& requestPool); +nn::GeneralResult clone(const Model& model); } // namespace aidl::android::hardware::neuralnetworks::utils diff --git a/neuralnetworks/aidl/utils/src/Utils.cpp b/neuralnetworks/aidl/utils/src/Utils.cpp index 04aa0e9eba..8d00e5926a 100644 --- a/neuralnetworks/aidl/utils/src/Utils.cpp +++ b/neuralnetworks/aidl/utils/src/Utils.cpp @@ -19,38 +19,77 @@ #include namespace aidl::android::hardware::neuralnetworks::utils { +namespace { using ::android::nn::GeneralResult; -GeneralResult copyModel(const Model& model) { - Model newModel{ +template +nn::GeneralResult> cloneVec(const std::vector& arguments) { + std::vector clonedObjects; + clonedObjects.reserve(arguments.size()); + for (const auto& argument : arguments) { + clonedObjects.push_back(NN_TRY(clone(argument))); + } + return clonedObjects; +} + +template +GeneralResult> clone(const std::vector& arguments) { + return cloneVec(arguments); +} + +} // namespace + +GeneralResult clone(const Memory& memory) { + common::NativeHandle nativeHandle; + nativeHandle.ints = memory.handle.ints; + nativeHandle.fds.reserve(memory.handle.fds.size()); + for (const auto& fd : memory.handle.fds) { + const int newFd = dup(fd.get()); + if (newFd < 0) { + return NN_ERROR() << "Couldn't dup a file descriptor"; + } + nativeHandle.fds.emplace_back(newFd); + } + return Memory{ + .handle = std::move(nativeHandle), + .size = memory.size, + .name = memory.name, + }; +} + +GeneralResult clone(const RequestMemoryPool& requestPool) { + using Tag = RequestMemoryPool::Tag; + switch (requestPool.getTag()) { + case Tag::pool: + return RequestMemoryPool::make(NN_TRY(clone(requestPool.get()))); + case Tag::token: + return RequestMemoryPool::make(requestPool.get()); + } + // Using explicit type conversion because std::variant inside the RequestMemoryPool confuses the + // compiler. + return (NN_ERROR() << "Unrecognized request pool tag: " << requestPool.getTag()) + . + operator GeneralResult(); +} + +GeneralResult clone(const Request& request) { + return Request{ + .inputs = request.inputs, + .outputs = request.outputs, + .pools = NN_TRY(clone(request.pools)), + }; +} + +GeneralResult clone(const Model& model) { + return Model{ .main = model.main, .referenced = model.referenced, .operandValues = model.operandValues, - .pools = {}, + .pools = NN_TRY(clone(model.pools)), .relaxComputationFloat32toFloat16 = model.relaxComputationFloat32toFloat16, .extensionNameToPrefix = model.extensionNameToPrefix, }; - newModel.pools.reserve(model.pools.size()); - for (const auto& pool : model.pools) { - common::NativeHandle nativeHandle; - nativeHandle.ints = pool.handle.ints; - nativeHandle.fds.reserve(pool.handle.fds.size()); - for (const auto& fd : pool.handle.fds) { - const int newFd = dup(fd.get()); - if (newFd == -1) { - return NN_ERROR() << "Couldn't dup a file descriptor."; - } - nativeHandle.fds.emplace_back(newFd); - } - Memory memory = { - .handle = std::move(nativeHandle), - .size = pool.size, - .name = pool.name, - }; - newModel.pools.push_back(std::move(memory)); - } - return newModel; } } // namespace aidl::android::hardware::neuralnetworks::utils diff --git a/neuralnetworks/aidl/vts/OWNERS b/neuralnetworks/aidl/vts/OWNERS new file mode 100644 index 0000000000..6719a5b3a2 --- /dev/null +++ b/neuralnetworks/aidl/vts/OWNERS @@ -0,0 +1,12 @@ +# Neuralnetworks team +butlermichael@google.com +dgross@google.com +jeanluc@google.com +levp@google.com +miaowang@google.com +mikie@google.com +mks@google.com +pszczepaniak@google.com +slavash@google.com +vddang@google.com +xusongw@google.com diff --git a/neuralnetworks/aidl/vts/functional/Android.bp b/neuralnetworks/aidl/vts/functional/Android.bp new file mode 100644 index 0000000000..aa7afbf6a7 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/Android.bp @@ -0,0 +1,68 @@ +// +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +cc_test { + name: "VtsHalNeuralnetworksTargetTest", + defaults: [ + "neuralnetworks_vts_functional_defaults", + "use_libaidlvintf_gtest_helper_static", + ], + srcs: [ + "BasicTests.cpp", + "Callbacks.cpp", + "CompilationCachingTests.cpp", + "GeneratedTestHarness.cpp", + "MemoryDomainTests.cpp", + "QualityOfServiceTests.cpp", + "TestAssertions.cpp", + "TestMain.cpp", + "Utils.cpp", + "ValidateModel.cpp", + "ValidateRequest.cpp", + "VtsHalNeuralnetworks.cpp", + ], + shared_libs: [ + "libbinder_ndk", + "libnativewindow", + "libvndksupport", + ], + static_libs: [ + "android.hardware.common-V2-ndk_platform", + "android.hardware.neuralnetworks-V1-ndk_platform", + "android.hidl.allocator@1.0", + "android.hidl.memory@1.0", + "libgmock", + "libhidlmemory", + "libneuralnetworks_generated_test_harness", + "libneuralnetworks_utils", + "libsync", + "neuralnetworks_utils_hal_aidl", + ], + whole_static_libs: [ + "neuralnetworks_generated_V1_0_example", + "neuralnetworks_generated_V1_1_example", + "neuralnetworks_generated_V1_2_example", + "neuralnetworks_generated_V1_3_example", + ], + header_libs: [ + "libbase_headers", + "libneuralnetworks_headers", + ], + test_suites: [ + "general-tests", + "vts", + ], +} diff --git a/neuralnetworks/aidl/vts/functional/AndroidTest.xml b/neuralnetworks/aidl/vts/functional/AndroidTest.xml new file mode 100644 index 0000000000..384d42078f --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/AndroidTest.xml @@ -0,0 +1,33 @@ + + + + diff --git a/neuralnetworks/aidl/vts/functional/BasicTests.cpp b/neuralnetworks/aidl/vts/functional/BasicTests.cpp new file mode 100644 index 0000000000..b2f4507c22 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/BasicTests.cpp @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" + +#include +#include +#include +#include +#include +#include + +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using implementation::PreparedModelCallback; + +// create device test +TEST_P(NeuralNetworksAidlTest, CreateDevice) {} + +// initialization +TEST_P(NeuralNetworksAidlTest, GetCapabilitiesTest) { + Capabilities capabilities; + const auto retStatus = kDevice->getCapabilities(&capabilities); + ASSERT_TRUE(retStatus.isOk()); + + auto isPositive = [](const PerformanceInfo& perf) { + return perf.execTime > 0.0f && perf.powerUsage > 0.0f; + }; + + EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceScalar)); + EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceTensor)); + const auto& opPerf = capabilities.operandPerformance; + EXPECT_TRUE( + std::all_of(opPerf.begin(), opPerf.end(), + [isPositive](const OperandPerformance& a) { return isPositive(a.info); })); + EXPECT_TRUE(std::is_sorted(opPerf.begin(), opPerf.end(), + [](const OperandPerformance& a, const OperandPerformance& b) { + return a.type < b.type; + })); + EXPECT_TRUE(std::all_of(opPerf.begin(), opPerf.end(), [](const OperandPerformance& a) { + return a.type != OperandType::SUBGRAPH; + })); + EXPECT_TRUE(isPositive(capabilities.ifPerformance)); + EXPECT_TRUE(isPositive(capabilities.whilePerformance)); +} + +// detect cycle +TEST_P(NeuralNetworksAidlTest, CycleTest) { + // opnd0 = TENSOR_FLOAT32 // model input + // opnd1 = TENSOR_FLOAT32 // model input + // opnd2 = INT32 // model input + // opnd3 = ADD(opnd0, opnd4, opnd2) + // opnd4 = ADD(opnd1, opnd3, opnd2) + // opnd5 = ADD(opnd4, opnd0, opnd2) // model output + // + // +-----+ + // | | + // v | + // 3 = ADD(0, 4, 2) | + // | | + // +----------+ | + // | | + // v | + // 4 = ADD(1, 3, 2) | + // | | + // +----------------+ + // | + // | + // +-------+ + // | + // v + // 5 = ADD(4, 0, 2) + + const std::vector operands = { + { + // operands[0] + .type = OperandType::TENSOR_FLOAT32, + .dimensions = {1}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::SUBGRAPH_INPUT, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + { + // operands[1] + .type = OperandType::TENSOR_FLOAT32, + .dimensions = {1}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::SUBGRAPH_INPUT, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + { + // operands[2] + .type = OperandType::INT32, + .dimensions = {}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::SUBGRAPH_INPUT, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + { + // operands[3] + .type = OperandType::TENSOR_FLOAT32, + .dimensions = {1}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::TEMPORARY_VARIABLE, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + { + // operands[4] + .type = OperandType::TENSOR_FLOAT32, + .dimensions = {1}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::TEMPORARY_VARIABLE, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + { + // operands[5] + .type = OperandType::TENSOR_FLOAT32, + .dimensions = {1}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::SUBGRAPH_OUTPUT, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }, + }; + + const std::vector operations = { + {.type = OperationType::ADD, .inputs = {0, 4, 2}, .outputs = {3}}, + {.type = OperationType::ADD, .inputs = {1, 3, 2}, .outputs = {4}}, + {.type = OperationType::ADD, .inputs = {4, 0, 2}, .outputs = {5}}, + }; + + Subgraph subgraph = { + .operands = operands, + .operations = operations, + .inputIndexes = {0, 1, 2}, + .outputIndexes = {5}, + }; + const Model model = { + .main = std::move(subgraph), + .referenced = {}, + .operandValues = {}, + .pools = {}, + }; + + // ensure that getSupportedOperations() checks model validity + std::vector supportedOps; + const auto supportedOpsStatus = kDevice->getSupportedOperations(model, &supportedOps); + ASSERT_FALSE(supportedOpsStatus.isOk()); + ASSERT_EQ(supportedOpsStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(supportedOpsStatus.getServiceSpecificError()), + ErrorStatus::INVALID_ARGUMENT); + + // ensure that prepareModel() checks model validity + auto preparedModelCallback = ndk::SharedRefBase::make(); + auto prepareLaunchStatus = + kDevice->prepareModel(model, ExecutionPreference::FAST_SINGLE_ANSWER, kDefaultPriority, + kNoDeadline, {}, {}, kEmptyCacheToken, preparedModelCallback); + // Note that preparation can fail for reasons other than an + // invalid model (invalid model should result in + // INVALID_ARGUMENT) -- for example, perhaps not all + // operations are supported, or perhaps the device hit some + // kind of capacity limit. + ASSERT_FALSE(prepareLaunchStatus.isOk()); + EXPECT_EQ(prepareLaunchStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + EXPECT_NE(static_cast(prepareLaunchStatus.getServiceSpecificError()), + ErrorStatus::NONE); + + EXPECT_NE(preparedModelCallback->getStatus(), ErrorStatus::NONE); + EXPECT_EQ(preparedModelCallback->getPreparedModel(), nullptr); +} + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/Callbacks.cpp b/neuralnetworks/aidl/vts/functional/Callbacks.cpp new file mode 100644 index 0000000000..ca2bb48a3e --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/Callbacks.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Callbacks" + +#include "Callbacks.h" + +#include +#include +#include + +namespace aidl::android::hardware::neuralnetworks::implementation { + +ndk::ScopedAStatus PreparedModelCallback::notify( + ErrorStatus errorStatus, const std::shared_ptr& preparedModel) { + { + std::lock_guard hold(mMutex); + // quick-return if object has already been notified + if (mNotified) { + return ndk::ScopedAStatus::ok(); + } + // store results and mark as notified + mErrorStatus = errorStatus; + mPreparedModel = preparedModel; + mNotified = true; + } + mCondition.notify_all(); + return ndk::ScopedAStatus::ok(); +} + +void PreparedModelCallback::wait() const { + std::unique_lock lock(mMutex); + mCondition.wait(lock, [this] { return mNotified; }); +} + +ErrorStatus PreparedModelCallback::getStatus() const { + wait(); + return mErrorStatus; +} + +std::shared_ptr PreparedModelCallback::getPreparedModel() const { + wait(); + return mPreparedModel; +} + +} // namespace aidl::android::hardware::neuralnetworks::implementation diff --git a/neuralnetworks/aidl/vts/functional/Callbacks.h b/neuralnetworks/aidl/vts/functional/Callbacks.h new file mode 100644 index 0000000000..0eb4d5f4a6 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/Callbacks.h @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_NEURALNETWORKS_AIDL_CALLBACKS_H +#define ANDROID_HARDWARE_NEURALNETWORKS_AIDL_CALLBACKS_H + +#include +#include +#include + +#include +#include +#include + +/* + * The Callback classes are used internally by the NeuralNetworks runtime to + * synchronize between different threads. An asynchronous task is launched + * paired with a callback object. When a client thread requires the output being + * generated by the asynchronous task, the client thread can wait for the result + * and be blocked until it has completed. Any wait may safely be called + * concurrently, even on the same callback object. When the asynchronous task + * has finished its workload, it must immediately call "notify". If the + * asynchronous task has failed to launch, the function that tried to launch the + * asynchronous task must immediately call "notify". This "notify" call + * awakens any client threads waiting on the callback object. + * + * These classes exist to enable synchronization across AIDL. When + * synchronization is only required in the same process, consider using + * std::future, std::mutex, std::condition_variable, or std::experimental::latch + * instead. + */ + +namespace aidl::android::hardware::neuralnetworks::implementation { + +/** + * The PreparedModelCallback class is used to receive the error status of + * preparing a model as well as the prepared model from a task executing + * asynchronously with respect to the runtime. If a calling thread calls wait + * or get* on a PreparedModelCallback object and the corresponding asynchronous + * task has not finished preparing the model, the calling thread will block + * until the asynchronous task has called notify. + * + * If the callback object is notified more than once, only the results of the + * first call to notify are used, and the results from subsequent calls are + * discarded. + * + * This callback object is passed as an argument to IDevice::prepareModel*. + */ +class PreparedModelCallback : public BnPreparedModelCallback { + public: + /** + * IPreparedModelCallback::notify marks the callback object with the return + * status of the asynchronous model preparation along with the prepared + * model, and allows all prior and future wait calls on the + * PreparedModelCallback object to proceed. + * + * IPreparedModelCallback::notify must be called on a given PreparedModelCallback object. + * + * If the callback object is notified more than once, only the results of + * the first call to notify are used, and the results from subsequent calls + * are discarded. + * + * @param status Error status returned from asynchronously preparing the + * model; will be: + * - NONE if the asynchronous preparation was successful + * - DEVICE_UNAVAILABLE if driver is offline or busy + * - GENERAL_FAILURE if there is an unspecified error + * - INVALID_ARGUMENT if the input model is invalid + * @param preparedModel Returned model that has been prepared for execution, + * nullptr if the model was unable to be prepared. + */ + ndk::ScopedAStatus notify(ErrorStatus status, + const std::shared_ptr& preparedModel) override; + + /** + * PreparedModelCallback::wait blocks until notify has been called on the + * callback object. + */ + void wait() const; + + /** + * Retrieves the error status returned from the asynchronous task launched + * by IDevice::prepareModel*. If IDevice::prepareModel* has not finished + * asynchronously preparing the model, this call will block until the + * asynchronous task notifies the object. + * + * @return status Error status returned from asynchronously preparing the + * model; will be: + * - NONE if the asynchronous preparation was successful + * - DEVICE_UNAVAILABLE if driver is offline or busy + * - GENERAL_FAILURE if there is an unspecified error + * - INVALID_ARGUMENT if the input model is invalid + */ + ErrorStatus getStatus() const; + + /** + * Retrieves the model that has been prepared for execution from the + * asynchronous task launched by IDevice::prepareModel*. If + * IDevice::prepareModel* has not finished asynchronously preparing the + * model, this call will block until the asynchronous task notifies the + * object. + * + * @return preparedModel Returned model that has been prepared for + * execution, nullptr if the model was unable to be prepared. + */ + std::shared_ptr getPreparedModel() const; + + private: + mutable std::mutex mMutex; + mutable std::condition_variable mCondition; + bool mNotified GUARDED_BY(mMutex) = false; + ErrorStatus mErrorStatus = ErrorStatus::GENERAL_FAILURE; + std::shared_ptr mPreparedModel; +}; + +} // namespace aidl::android::hardware::neuralnetworks::implementation + +#endif // ANDROID_HARDWARE_NEURALNETWORKS_AIDL_CALLBACKS_H diff --git a/neuralnetworks/aidl/vts/functional/CompilationCachingTests.cpp b/neuralnetworks/aidl/vts/functional/CompilationCachingTests.cpp new file mode 100644 index 0000000000..e0b529f280 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/CompilationCachingTests.cpp @@ -0,0 +1,1177 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "MemoryUtils.h" +#include "TestHarness.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +// Forward declaration of the mobilenet generated test models in +// frameworks/ml/nn/runtime/test/generated/. +namespace generated_tests::mobilenet_224_gender_basic_fixed { +const test_helper::TestModel& get_test_model(); +} // namespace generated_tests::mobilenet_224_gender_basic_fixed + +namespace generated_tests::mobilenet_quantized { +const test_helper::TestModel& get_test_model(); +} // namespace generated_tests::mobilenet_quantized + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using namespace test_helper; +using implementation::PreparedModelCallback; + +namespace float32_model { + +constexpr auto get_test_model = generated_tests::mobilenet_224_gender_basic_fixed::get_test_model; + +} // namespace float32_model + +namespace quant8_model { + +constexpr auto get_test_model = generated_tests::mobilenet_quantized::get_test_model; + +} // namespace quant8_model + +namespace { + +enum class AccessMode { READ_WRITE, READ_ONLY, WRITE_ONLY }; + +// Creates cache handles based on provided file groups. +// The outer vector corresponds to handles and the inner vector is for fds held by each handle. +void createCacheFds(const std::vector& files, const std::vector& mode, + std::vector* fds) { + fds->clear(); + fds->reserve(files.size()); + for (uint32_t i = 0; i < files.size(); i++) { + const auto& file = files[i]; + int fd; + if (mode[i] == AccessMode::READ_ONLY) { + fd = open(file.c_str(), O_RDONLY); + } else if (mode[i] == AccessMode::WRITE_ONLY) { + fd = open(file.c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + } else if (mode[i] == AccessMode::READ_WRITE) { + fd = open(file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + } else { + FAIL(); + } + ASSERT_GE(fd, 0); + fds->emplace_back(fd); + } +} + +void createCacheFds(const std::vector& files, AccessMode mode, + std::vector* fds) { + createCacheFds(files, std::vector(files.size(), mode), fds); +} + +// Create a chain of broadcast operations. The second operand is always constant tensor [1]. +// For simplicity, activation scalar is shared. The second operand is not shared +// in the model to let driver maintain a non-trivial size of constant data and the corresponding +// data locations in cache. +// +// --------- activation -------- +// ↓ ↓ ↓ ↓ +// E.g. input -> ADD -> ADD -> ADD -> ... -> ADD -> output +// ↑ ↑ ↑ ↑ +// [1] [1] [1] [1] +// +// This function assumes the operation is either ADD or MUL. +template +TestModel createLargeTestModelImpl(TestOperationType op, uint32_t len) { + EXPECT_TRUE(op == TestOperationType::ADD || op == TestOperationType::MUL); + + // Model operations and operands. + std::vector operations(len); + std::vector operands(len * 2 + 2); + + // The activation scalar, value = 0. + operands[0] = { + .type = TestOperandType::INT32, + .dimensions = {}, + .numberOfConsumers = len, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY, + .data = TestBuffer::createFromVector({0}), + }; + + // The buffer value of the constant second operand. The logical value is always 1.0f. + CppType bufferValue; + // The scale of the first and second operand. + float scale1, scale2; + if (operandType == TestOperandType::TENSOR_FLOAT32) { + bufferValue = 1.0f; + scale1 = 0.0f; + scale2 = 0.0f; + } else if (op == TestOperationType::ADD) { + bufferValue = 1; + scale1 = 1.0f; + scale2 = 1.0f; + } else { + // To satisfy the constraint on quant8 MUL: input0.scale * input1.scale < output.scale, + // set input1 to have scale = 0.5f and bufferValue = 2, i.e. 1.0f in floating point. + bufferValue = 2; + scale1 = 1.0f; + scale2 = 0.5f; + } + + for (uint32_t i = 0; i < len; i++) { + const uint32_t firstInputIndex = i * 2 + 1; + const uint32_t secondInputIndex = firstInputIndex + 1; + const uint32_t outputIndex = secondInputIndex + 1; + + // The first operation input. + operands[firstInputIndex] = { + .type = operandType, + .dimensions = {1}, + .numberOfConsumers = 1, + .scale = scale1, + .zeroPoint = 0, + .lifetime = (i == 0 ? TestOperandLifeTime::MODEL_INPUT + : TestOperandLifeTime::TEMPORARY_VARIABLE), + .data = (i == 0 ? TestBuffer::createFromVector({1}) : TestBuffer()), + }; + + // The second operation input, value = 1. + operands[secondInputIndex] = { + .type = operandType, + .dimensions = {1}, + .numberOfConsumers = 1, + .scale = scale2, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY, + .data = TestBuffer::createFromVector({bufferValue}), + }; + + // The operation. All operations share the same activation scalar. + // The output operand is created as an input in the next iteration of the loop, in the case + // of all but the last member of the chain; and after the loop as a model output, in the + // case of the last member of the chain. + operations[i] = { + .type = op, + .inputs = {firstInputIndex, secondInputIndex, /*activation scalar*/ 0}, + .outputs = {outputIndex}, + }; + } + + // For TestOperationType::ADD, output = 1 + 1 * len = len + 1 + // For TestOperationType::MUL, output = 1 * 1 ^ len = 1 + CppType outputResult = static_cast(op == TestOperationType::ADD ? len + 1u : 1u); + + // The model output. + operands.back() = { + .type = operandType, + .dimensions = {1}, + .numberOfConsumers = 0, + .scale = scale1, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::MODEL_OUTPUT, + .data = TestBuffer::createFromVector({outputResult}), + }; + + return { + .main = {.operands = std::move(operands), + .operations = std::move(operations), + .inputIndexes = {1}, + .outputIndexes = {len * 2 + 1}}, + .isRelaxed = false, + }; +} + +} // namespace + +// Tag for the compilation caching tests. +class CompilationCachingTestBase : public testing::Test { + protected: + CompilationCachingTestBase(std::shared_ptr device, OperandType type) + : kDevice(std::move(device)), kOperandType(type) {} + + void SetUp() override { + testing::Test::SetUp(); + ASSERT_NE(kDevice.get(), nullptr); + + // Create cache directory. The cache directory and a temporary cache file is always created + // to test the behavior of prepareModelFromCache, even when caching is not supported. + char cacheDirTemp[] = "/data/local/tmp/TestCompilationCachingXXXXXX"; + char* cacheDir = mkdtemp(cacheDirTemp); + ASSERT_NE(cacheDir, nullptr); + mCacheDir = cacheDir; + mCacheDir.push_back('/'); + + NumberOfCacheFiles numCacheFiles; + const auto ret = kDevice->getNumberOfCacheFilesNeeded(&numCacheFiles); + ASSERT_TRUE(ret.isOk()); + + mNumModelCache = numCacheFiles.numModelCache; + mNumDataCache = numCacheFiles.numDataCache; + ASSERT_GE(mNumModelCache, 0) << "Invalid numModelCache: " << mNumModelCache; + ASSERT_GE(mNumDataCache, 0) << "Invalid numDataCache: " << mNumDataCache; + mIsCachingSupported = mNumModelCache > 0 || mNumDataCache > 0; + + // Create empty cache files. + mTmpCache = mCacheDir + "tmp"; + for (uint32_t i = 0; i < mNumModelCache; i++) { + mModelCache.push_back({mCacheDir + "model" + std::to_string(i)}); + } + for (uint32_t i = 0; i < mNumDataCache; i++) { + mDataCache.push_back({mCacheDir + "data" + std::to_string(i)}); + } + // Placeholder handles, use AccessMode::WRITE_ONLY for createCacheFds to create files. + std::vector modelHandle, dataHandle, tmpHandle; + createCacheFds(mModelCache, AccessMode::WRITE_ONLY, &modelHandle); + createCacheFds(mDataCache, AccessMode::WRITE_ONLY, &dataHandle); + createCacheFds({mTmpCache}, AccessMode::WRITE_ONLY, &tmpHandle); + + if (!mIsCachingSupported) { + LOG(INFO) << "NN VTS: Early termination of test because vendor service does not " + "support compilation caching."; + std::cout << "[ ] Early termination of test because vendor service does not " + "support compilation caching." + << std::endl; + } + } + + void TearDown() override { + // If the test passes, remove the tmp directory. Otherwise, keep it for debugging purposes. + if (!testing::Test::HasFailure()) { + // Recursively remove the cache directory specified by mCacheDir. + auto callback = [](const char* entry, const struct stat*, int, struct FTW*) { + return remove(entry); + }; + nftw(mCacheDir.c_str(), callback, 128, FTW_DEPTH | FTW_MOUNT | FTW_PHYS); + } + testing::Test::TearDown(); + } + + // Model and examples creators. According to kOperandType, the following methods will return + // either float32 model/examples or the quant8 variant. + TestModel createTestModel() { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return float32_model::get_test_model(); + } else { + return quant8_model::get_test_model(); + } + } + + TestModel createLargeTestModel(OperationType op, uint32_t len) { + if (kOperandType == OperandType::TENSOR_FLOAT32) { + return createLargeTestModelImpl( + static_cast(op), len); + } else { + return createLargeTestModelImpl( + static_cast(op), len); + } + } + + // See if the service can handle the model. + bool isModelFullySupported(const Model& model) { + std::vector supportedOps; + const auto supportedCall = kDevice->getSupportedOperations(model, &supportedOps); + EXPECT_TRUE(supportedCall.isOk()); + EXPECT_EQ(supportedOps.size(), model.main.operations.size()); + if (!supportedCall.isOk() || supportedOps.size() != model.main.operations.size()) { + return false; + } + return std::all_of(supportedOps.begin(), supportedOps.end(), + [](bool valid) { return valid; }); + } + + void saveModelToCache(const Model& model, + const std::vector& modelCache, + const std::vector& dataCache, + std::shared_ptr* preparedModel = nullptr) { + if (preparedModel != nullptr) *preparedModel = nullptr; + + // Launch prepare model. + std::shared_ptr preparedModelCallback = + ndk::SharedRefBase::make(); + std::vector cacheToken(std::begin(mToken), std::end(mToken)); + const auto prepareLaunchStatus = kDevice->prepareModel( + model, ExecutionPreference::FAST_SINGLE_ANSWER, kDefaultPriority, kNoDeadline, + modelCache, dataCache, cacheToken, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()); + + // Retrieve prepared model. + preparedModelCallback->wait(); + ASSERT_EQ(preparedModelCallback->getStatus(), ErrorStatus::NONE); + if (preparedModel != nullptr) { + *preparedModel = preparedModelCallback->getPreparedModel(); + } + } + + bool checkEarlyTermination(ErrorStatus status) { + if (status == ErrorStatus::GENERAL_FAILURE) { + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "save the prepared model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "save the prepared model that it does not support." + << std::endl; + return true; + } + return false; + } + + bool checkEarlyTermination(const Model& model) { + if (!isModelFullySupported(model)) { + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + return true; + } + return false; + } + + void prepareModelFromCache(const std::vector& modelCache, + const std::vector& dataCache, + std::shared_ptr* preparedModel, + ErrorStatus* status) { + // Launch prepare model from cache. + std::shared_ptr preparedModelCallback = + ndk::SharedRefBase::make(); + std::vector cacheToken(std::begin(mToken), std::end(mToken)); + const auto prepareLaunchStatus = kDevice->prepareModelFromCache( + kNoDeadline, modelCache, dataCache, cacheToken, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk() || + prepareLaunchStatus.getExceptionCode() == EX_SERVICE_SPECIFIC) + << "prepareLaunchStatus: " << prepareLaunchStatus.getDescription(); + if (!prepareLaunchStatus.isOk()) { + *preparedModel = nullptr; + *status = static_cast(prepareLaunchStatus.getServiceSpecificError()); + return; + } + + // Retrieve prepared model. + preparedModelCallback->wait(); + *status = preparedModelCallback->getStatus(); + *preparedModel = preparedModelCallback->getPreparedModel(); + } + + // Absolute path to the temporary cache directory. + std::string mCacheDir; + + // Groups of file paths for model and data cache in the tmp cache directory, initialized with + // size = mNum{Model|Data}Cache. The outer vector corresponds to handles and the inner vector is + // for fds held by each handle. + std::vector mModelCache; + std::vector mDataCache; + + // A separate temporary file path in the tmp cache directory. + std::string mTmpCache; + + uint8_t mToken[static_cast(IDevice::BYTE_SIZE_OF_CACHE_TOKEN)] = {}; + uint32_t mNumModelCache; + uint32_t mNumDataCache; + uint32_t mIsCachingSupported; + + const std::shared_ptr kDevice; + // The primary data type of the testModel. + const OperandType kOperandType; +}; + +using CompilationCachingTestParam = std::tuple; + +// A parameterized fixture of CompilationCachingTestBase. Every test will run twice, with the first +// pass running with float32 models and the second pass running with quant8 models. +class CompilationCachingTest : public CompilationCachingTestBase, + public testing::WithParamInterface { + protected: + CompilationCachingTest() + : CompilationCachingTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} +}; + +TEST_P(CompilationCachingTest, CacheSavingAndRetrieval) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + std::shared_ptr preparedModel = nullptr; + + // Save the compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(model, modelCache, dataCache); + } + + // Retrieve preparedModel from cache. + { + preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (!mIsCachingSupported) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + return; + } else if (checkEarlyTermination(status)) { + ASSERT_EQ(preparedModel, nullptr); + return; + } else { + ASSERT_EQ(status, ErrorStatus::NONE); + ASSERT_NE(preparedModel, nullptr); + } + } + + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); +} + +TEST_P(CompilationCachingTest, CacheSavingAndRetrievalNonZeroOffset) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + std::shared_ptr preparedModel = nullptr; + + // Save the compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + uint8_t placeholderBytes[] = {0, 0}; + // Write a placeholder integer to the cache. + // The driver should be able to handle non-empty cache and non-zero fd offset. + for (uint32_t i = 0; i < modelCache.size(); i++) { + ASSERT_EQ(write(modelCache[i].get(), &placeholderBytes, sizeof(placeholderBytes)), + sizeof(placeholderBytes)); + } + for (uint32_t i = 0; i < dataCache.size(); i++) { + ASSERT_EQ(write(dataCache[i].get(), &placeholderBytes, sizeof(placeholderBytes)), + sizeof(placeholderBytes)); + } + saveModelToCache(model, modelCache, dataCache); + } + + // Retrieve preparedModel from cache. + { + preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + uint8_t placeholderByte = 0; + // Advance the offset of each handle by one byte. + // The driver should be able to handle non-zero fd offset. + for (uint32_t i = 0; i < modelCache.size(); i++) { + ASSERT_GE(read(modelCache[i].get(), &placeholderByte, 1), 0); + } + for (uint32_t i = 0; i < dataCache.size(); i++) { + ASSERT_GE(read(dataCache[i].get(), &placeholderByte, 1), 0); + } + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (!mIsCachingSupported) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + return; + } else if (checkEarlyTermination(status)) { + ASSERT_EQ(preparedModel, nullptr); + return; + } else { + ASSERT_EQ(status, ErrorStatus::NONE); + ASSERT_NE(preparedModel, nullptr); + } + } + + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); +} + +TEST_P(CompilationCachingTest, SaveToCacheInvalidNumCache) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + + // Test with number of model cache files greater than mNumModelCache. + { + std::vector modelCache, dataCache; + // Pass an additional cache file for model cache. + mModelCache.push_back({mTmpCache}); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mModelCache.pop_back(); + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of model cache files smaller than mNumModelCache. + if (mModelCache.size() > 0) { + std::vector modelCache, dataCache; + // Pop out the last cache file. + auto tmp = mModelCache.back(); + mModelCache.pop_back(); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mModelCache.push_back(tmp); + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of data cache files greater than mNumDataCache. + { + std::vector modelCache, dataCache; + // Pass an additional cache file for data cache. + mDataCache.push_back({mTmpCache}); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mDataCache.pop_back(); + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of data cache files smaller than mNumDataCache. + if (mDataCache.size() > 0) { + std::vector modelCache, dataCache; + // Pop out the last cache file. + auto tmp = mDataCache.back(); + mDataCache.pop_back(); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mDataCache.push_back(tmp); + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } +} + +TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidNumCache) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + + // Save the compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(model, modelCache, dataCache); + } + + // Test with number of model cache files greater than mNumModelCache. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + mModelCache.push_back({mTmpCache}); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mModelCache.pop_back(); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of model cache files smaller than mNumModelCache. + if (mModelCache.size() > 0) { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + auto tmp = mModelCache.back(); + mModelCache.pop_back(); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mModelCache.push_back(tmp); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of data cache files greater than mNumDataCache. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + mDataCache.push_back({mTmpCache}); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mDataCache.pop_back(); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Test with number of data cache files smaller than mNumDataCache. + if (mDataCache.size() > 0) { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + auto tmp = mDataCache.back(); + mDataCache.pop_back(); + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + mDataCache.push_back(tmp); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::GENERAL_FAILURE) { + ASSERT_EQ(status, ErrorStatus::INVALID_ARGUMENT); + } + ASSERT_EQ(preparedModel, nullptr); + } +} + +TEST_P(CompilationCachingTest, SaveToCacheInvalidAccessMode) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + std::vector modelCacheMode(mNumModelCache, AccessMode::READ_WRITE); + std::vector dataCacheMode(mNumDataCache, AccessMode::READ_WRITE); + + // Go through each handle in model cache, test with invalid access mode. + for (uint32_t i = 0; i < mNumModelCache; i++) { + std::vector modelCache, dataCache; + modelCacheMode[i] = AccessMode::READ_ONLY; + createCacheFds(mModelCache, modelCacheMode, &modelCache); + createCacheFds(mDataCache, dataCacheMode, &dataCache); + modelCacheMode[i] = AccessMode::READ_WRITE; + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } + + // Go through each handle in data cache, test with invalid access mode. + for (uint32_t i = 0; i < mNumDataCache; i++) { + std::vector modelCache, dataCache; + dataCacheMode[i] = AccessMode::READ_ONLY; + createCacheFds(mModelCache, modelCacheMode, &modelCache); + createCacheFds(mDataCache, dataCacheMode, &dataCache); + dataCacheMode[i] = AccessMode::READ_WRITE; + std::shared_ptr preparedModel = nullptr; + saveModelToCache(model, modelCache, dataCache, &preparedModel); + ASSERT_NE(preparedModel, nullptr); + // Execute and verify results. + EvaluatePreparedModel(kDevice, preparedModel, testModel, /*testKind=*/TestKind::GENERAL); + // Check if prepareModelFromCache fails. + preparedModel = nullptr; + ErrorStatus status; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + if (status != ErrorStatus::INVALID_ARGUMENT) { + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + } + ASSERT_EQ(preparedModel, nullptr); + } +} + +TEST_P(CompilationCachingTest, PrepareModelFromCacheInvalidAccessMode) { + // Create test HIDL model and compile. + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + std::vector modelCacheMode(mNumModelCache, AccessMode::READ_WRITE); + std::vector dataCacheMode(mNumDataCache, AccessMode::READ_WRITE); + + // Save the compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(model, modelCache, dataCache); + } + + // Go through each handle in model cache, test with invalid access mode. + for (uint32_t i = 0; i < mNumModelCache; i++) { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + modelCacheMode[i] = AccessMode::WRITE_ONLY; + createCacheFds(mModelCache, modelCacheMode, &modelCache); + createCacheFds(mDataCache, dataCacheMode, &dataCache); + modelCacheMode[i] = AccessMode::READ_WRITE; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + } + + // Go through each handle in data cache, test with invalid access mode. + for (uint32_t i = 0; i < mNumDataCache; i++) { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + dataCacheMode[i] = AccessMode::WRITE_ONLY; + createCacheFds(mModelCache, modelCacheMode, &modelCache); + createCacheFds(mDataCache, dataCacheMode, &dataCache); + dataCacheMode[i] = AccessMode::READ_WRITE; + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + } +} + +// Copy file contents between files. +// The vector sizes must match. +static void copyCacheFiles(const std::vector& from, + const std::vector& to) { + constexpr size_t kBufferSize = 1000000; + uint8_t buffer[kBufferSize]; + + ASSERT_EQ(from.size(), to.size()); + for (uint32_t i = 0; i < from.size(); i++) { + int fromFd = open(from[i].c_str(), O_RDONLY); + int toFd = open(to[i].c_str(), O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + ASSERT_GE(fromFd, 0); + ASSERT_GE(toFd, 0); + + ssize_t readBytes; + while ((readBytes = read(fromFd, &buffer, kBufferSize)) > 0) { + ASSERT_EQ(write(toFd, &buffer, readBytes), readBytes); + } + ASSERT_GE(readBytes, 0); + + close(fromFd); + close(toFd); + } +} + +// Number of operations in the large test model. +constexpr uint32_t kLargeModelSize = 100; +constexpr uint32_t kNumIterationsTOCTOU = 100; + +TEST_P(CompilationCachingTest, SaveToCache_TOCTOU) { + if (!mIsCachingSupported) return; + + // Create test models and check if fully supported by the service. + const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); + const Model modelMul = createModel(testModelMul); + if (checkEarlyTermination(modelMul)) return; + const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); + const Model modelAdd = createModel(testModelAdd); + if (checkEarlyTermination(modelAdd)) return; + + // Save the modelMul compilation to cache. + auto modelCacheMul = mModelCache; + for (auto& cache : modelCacheMul) { + cache.append("_mul"); + } + { + std::vector modelCache, dataCache; + createCacheFds(modelCacheMul, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(modelMul, modelCache, dataCache); + } + + // Use a different token for modelAdd. + mToken[0]++; + + // This test is probabilistic, so we run it multiple times. + for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // Save the modelAdd compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + + // Spawn a thread to copy the cache content concurrently while saving to cache. + std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); + saveModelToCache(modelAdd, modelCache, dataCache); + thread.join(); + } + + // Retrieve preparedModel from cache. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + + // The preparation may fail or succeed, but must not crash. If the preparation succeeds, + // the prepared model must be executed with the correct result and not crash. + if (status != ErrorStatus::NONE) { + ASSERT_EQ(preparedModel, nullptr); + } else { + ASSERT_NE(preparedModel, nullptr); + EvaluatePreparedModel(kDevice, preparedModel, testModelAdd, + /*testKind=*/TestKind::GENERAL); + } + } + } +} + +TEST_P(CompilationCachingTest, PrepareFromCache_TOCTOU) { + if (!mIsCachingSupported) return; + + // Create test models and check if fully supported by the service. + const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); + const Model modelMul = createModel(testModelMul); + if (checkEarlyTermination(modelMul)) return; + const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); + const Model modelAdd = createModel(testModelAdd); + if (checkEarlyTermination(modelAdd)) return; + + // Save the modelMul compilation to cache. + auto modelCacheMul = mModelCache; + for (auto& cache : modelCacheMul) { + cache.append("_mul"); + } + { + std::vector modelCache, dataCache; + createCacheFds(modelCacheMul, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(modelMul, modelCache, dataCache); + } + + // Use a different token for modelAdd. + mToken[0]++; + + // This test is probabilistic, so we run it multiple times. + for (uint32_t i = 0; i < kNumIterationsTOCTOU; i++) { + // Save the modelAdd compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(modelAdd, modelCache, dataCache); + } + + // Retrieve preparedModel from cache. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + + // Spawn a thread to copy the cache content concurrently while preparing from cache. + std::thread thread(copyCacheFiles, std::cref(modelCacheMul), std::cref(mModelCache)); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + thread.join(); + + // The preparation may fail or succeed, but must not crash. If the preparation succeeds, + // the prepared model must be executed with the correct result and not crash. + if (status != ErrorStatus::NONE) { + ASSERT_EQ(preparedModel, nullptr); + } else { + ASSERT_NE(preparedModel, nullptr); + EvaluatePreparedModel(kDevice, preparedModel, testModelAdd, + /*testKind=*/TestKind::GENERAL); + } + } + } +} + +TEST_P(CompilationCachingTest, ReplaceSecuritySensitiveCache) { + if (!mIsCachingSupported) return; + + // Create test models and check if fully supported by the service. + const TestModel testModelMul = createLargeTestModel(OperationType::MUL, kLargeModelSize); + const Model modelMul = createModel(testModelMul); + if (checkEarlyTermination(modelMul)) return; + const TestModel testModelAdd = createLargeTestModel(OperationType::ADD, kLargeModelSize); + const Model modelAdd = createModel(testModelAdd); + if (checkEarlyTermination(modelAdd)) return; + + // Save the modelMul compilation to cache. + auto modelCacheMul = mModelCache; + for (auto& cache : modelCacheMul) { + cache.append("_mul"); + } + { + std::vector modelCache, dataCache; + createCacheFds(modelCacheMul, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(modelMul, modelCache, dataCache); + } + + // Use a different token for modelAdd. + mToken[0]++; + + // Save the modelAdd compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(modelAdd, modelCache, dataCache); + } + + // Replace the model cache of modelAdd with modelMul. + copyCacheFiles(modelCacheMul, mModelCache); + + // Retrieve the preparedModel from cache, expect failure. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + } +} + +// TODO(b/179270601): restore kNamedDeviceChoices. +static const auto kOperandTypeChoices = + testing::Values(OperandType::TENSOR_FLOAT32, OperandType::TENSOR_QUANT8_ASYMM); + +std::string printCompilationCachingTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType] = info.param; + const std::string type = (operandType == OperandType::TENSOR_FLOAT32 ? "float32" : "quant8"); + return gtestCompliantName(getName(namedDevice) + "_" + type); +} + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CompilationCachingTest); +INSTANTIATE_TEST_SUITE_P(TestCompilationCaching, CompilationCachingTest, + testing::Combine(testing::ValuesIn(getNamedDevices()), + kOperandTypeChoices), + printCompilationCachingTest); + +using CompilationCachingSecurityTestParam = std::tuple; + +class CompilationCachingSecurityTest + : public CompilationCachingTestBase, + public testing::WithParamInterface { + protected: + CompilationCachingSecurityTest() + : CompilationCachingTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} + + void SetUp() { + CompilationCachingTestBase::SetUp(); + generator.seed(kSeed); + } + + // Get a random integer within a closed range [lower, upper]. + template + T getRandomInt(T lower, T upper) { + std::uniform_int_distribution dis(lower, upper); + return dis(generator); + } + + // Randomly flip one single bit of the cache entry. + void flipOneBitOfCache(const std::string& filename, bool* skip) { + FILE* pFile = fopen(filename.c_str(), "r+"); + ASSERT_EQ(fseek(pFile, 0, SEEK_END), 0); + long int fileSize = ftell(pFile); + if (fileSize == 0) { + fclose(pFile); + *skip = true; + return; + } + ASSERT_EQ(fseek(pFile, getRandomInt(0l, fileSize - 1), SEEK_SET), 0); + int readByte = fgetc(pFile); + ASSERT_NE(readByte, EOF); + ASSERT_EQ(fseek(pFile, -1, SEEK_CUR), 0); + ASSERT_NE(fputc(static_cast(readByte) ^ (1U << getRandomInt(0, 7)), pFile), EOF); + fclose(pFile); + *skip = false; + } + + // Randomly append bytes to the cache entry. + void appendBytesToCache(const std::string& filename, bool* skip) { + FILE* pFile = fopen(filename.c_str(), "a"); + uint32_t appendLength = getRandomInt(1, 256); + for (uint32_t i = 0; i < appendLength; i++) { + ASSERT_NE(fputc(getRandomInt(0, 255), pFile), EOF); + } + fclose(pFile); + *skip = false; + } + + enum class ExpectedResult { GENERAL_FAILURE, NOT_CRASH }; + + // Test if the driver behaves as expected when given corrupted cache or token. + // The modifier will be invoked after save to cache but before prepare from cache. + // The modifier accepts one pointer argument "skip" as the returning value, indicating + // whether the test should be skipped or not. + void testCorruptedCache(ExpectedResult expected, std::function modifier) { + const TestModel& testModel = createTestModel(); + const Model model = createModel(testModel); + if (checkEarlyTermination(model)) return; + + // Save the compilation to cache. + { + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + saveModelToCache(model, modelCache, dataCache); + } + + bool skip = false; + modifier(&skip); + if (skip) return; + + // Retrieve preparedModel from cache. + { + std::shared_ptr preparedModel = nullptr; + ErrorStatus status; + std::vector modelCache, dataCache; + createCacheFds(mModelCache, AccessMode::READ_WRITE, &modelCache); + createCacheFds(mDataCache, AccessMode::READ_WRITE, &dataCache); + prepareModelFromCache(modelCache, dataCache, &preparedModel, &status); + + switch (expected) { + case ExpectedResult::GENERAL_FAILURE: + ASSERT_EQ(status, ErrorStatus::GENERAL_FAILURE); + ASSERT_EQ(preparedModel, nullptr); + break; + case ExpectedResult::NOT_CRASH: + ASSERT_EQ(preparedModel == nullptr, status != ErrorStatus::NONE); + break; + default: + FAIL(); + } + } + } + + const uint32_t kSeed = std::get(GetParam()); + std::mt19937 generator; +}; + +TEST_P(CompilationCachingSecurityTest, CorruptedModelCache) { + if (!mIsCachingSupported) return; + for (uint32_t i = 0; i < mNumModelCache; i++) { + testCorruptedCache(ExpectedResult::GENERAL_FAILURE, + [this, i](bool* skip) { flipOneBitOfCache(mModelCache[i], skip); }); + } +} + +TEST_P(CompilationCachingSecurityTest, WrongLengthModelCache) { + if (!mIsCachingSupported) return; + for (uint32_t i = 0; i < mNumModelCache; i++) { + testCorruptedCache(ExpectedResult::GENERAL_FAILURE, + [this, i](bool* skip) { appendBytesToCache(mModelCache[i], skip); }); + } +} + +TEST_P(CompilationCachingSecurityTest, CorruptedDataCache) { + if (!mIsCachingSupported) return; + for (uint32_t i = 0; i < mNumDataCache; i++) { + testCorruptedCache(ExpectedResult::NOT_CRASH, + [this, i](bool* skip) { flipOneBitOfCache(mDataCache[i], skip); }); + } +} + +TEST_P(CompilationCachingSecurityTest, WrongLengthDataCache) { + if (!mIsCachingSupported) return; + for (uint32_t i = 0; i < mNumDataCache; i++) { + testCorruptedCache(ExpectedResult::NOT_CRASH, + [this, i](bool* skip) { appendBytesToCache(mDataCache[i], skip); }); + } +} + +TEST_P(CompilationCachingSecurityTest, WrongToken) { + if (!mIsCachingSupported) return; + testCorruptedCache(ExpectedResult::GENERAL_FAILURE, [this](bool* skip) { + // Randomly flip one single bit in mToken. + uint32_t ind = + getRandomInt(0u, static_cast(IDevice::BYTE_SIZE_OF_CACHE_TOKEN) - 1); + mToken[ind] ^= (1U << getRandomInt(0, 7)); + *skip = false; + }); +} + +std::string printCompilationCachingSecurityTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType, seed] = info.param; + const std::string type = (operandType == OperandType::TENSOR_FLOAT32 ? "float32" : "quant8"); + return gtestCompliantName(getName(namedDevice) + "_" + type + "_" + std::to_string(seed)); +} + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(CompilationCachingSecurityTest); +INSTANTIATE_TEST_SUITE_P(TestCompilationCaching, CompilationCachingSecurityTest, + testing::Combine(testing::ValuesIn(getNamedDevices()), kOperandTypeChoices, + testing::Range(0U, 10U)), + printCompilationCachingSecurityTest); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.cpp b/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.cpp new file mode 100644 index 0000000000..86d5f3f8d3 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.cpp @@ -0,0 +1,925 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "GeneratedTestHarness.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "Callbacks.h" +#include "TestHarness.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +namespace nn = ::android::nn; +using namespace test_helper; +using implementation::PreparedModelCallback; + +namespace { + +enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE }; + +struct TestConfig { + Executor executor; + bool measureTiming; + OutputType outputType; + MemoryType memoryType; + // `reportSkipping` indicates if a test should print an info message in case + // it is skipped. The field is set to true by default and is set to false in + // quantization coupling tests to suppress skipping a test + bool reportSkipping; + TestConfig(Executor executor, bool measureTiming, OutputType outputType, MemoryType memoryType) + : executor(executor), + measureTiming(measureTiming), + outputType(outputType), + memoryType(memoryType), + reportSkipping(true) {} + TestConfig(Executor executor, bool measureTiming, OutputType outputType, MemoryType memoryType, + bool reportSkipping) + : executor(executor), + measureTiming(measureTiming), + outputType(outputType), + memoryType(memoryType), + reportSkipping(reportSkipping) {} +}; + +enum class IOType { INPUT, OUTPUT }; + +class DeviceMemoryAllocator { + public: + DeviceMemoryAllocator(const std::shared_ptr& device, + const std::shared_ptr& preparedModel, + const TestModel& testModel) + : kDevice(device), kPreparedModel(preparedModel), kTestModel(testModel) {} + + // Allocate device memory for a target input/output operand. + // Return {IBuffer object, token} if successful. + // Return {nullptr, 0} if device memory is not supported. + template + std::pair, int32_t> allocate(uint32_t index) { + std::pair, int32_t> buffer; + allocateInternal(index, &buffer); + return buffer; + } + + private: + template + void allocateInternal(int32_t index, std::pair, int32_t>* result) { + ASSERT_NE(result, nullptr); + + // Prepare arguments. + BufferRole role = {.modelIndex = 0, .ioIndex = index, .frequency = 1.0f}; + std::vector inputRoles, outputRoles; + if constexpr (ioType == IOType::INPUT) { + inputRoles = {role}; + } else { + outputRoles = {role}; + } + + // Allocate device memory. + DeviceBuffer buffer; + IPreparedModelParcel parcel; + parcel.preparedModel = kPreparedModel; + const auto ret = kDevice->allocate({}, {parcel}, inputRoles, outputRoles, &buffer); + + // Check allocation results. + if (ret.isOk()) { + ASSERT_NE(buffer.buffer, nullptr); + ASSERT_GT(buffer.token, 0); + } else { + ASSERT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(ret.getServiceSpecificError()), + ErrorStatus::GENERAL_FAILURE); + buffer.buffer = nullptr; + buffer.token = 0; + } + + // Initialize input data from TestBuffer. + if constexpr (ioType == IOType::INPUT) { + if (buffer.buffer != nullptr) { + // TestBuffer -> Shared memory. + const auto& testBuffer = + kTestModel.main.operands[kTestModel.main.inputIndexes[index]].data; + ASSERT_GT(testBuffer.size(), 0); + const auto sharedMemory = nn::createSharedMemory(testBuffer.size()).value(); + const auto memory = utils::convert(sharedMemory).value(); + const auto mapping = nn::map(sharedMemory).value(); + uint8_t* inputPtr = static_cast(std::get(mapping.pointer)); + ASSERT_NE(inputPtr, nullptr); + const uint8_t* begin = testBuffer.get(); + const uint8_t* end = begin + testBuffer.size(); + std::copy(begin, end, inputPtr); + + // Shared memory -> IBuffer. + auto ret = buffer.buffer->copyFrom(memory, {}); + ASSERT_TRUE(ret.isOk()); + } + } + *result = {std::move(buffer.buffer), buffer.token}; + } + + const std::shared_ptr kDevice; + const std::shared_ptr kPreparedModel; + const TestModel& kTestModel; +}; + +Subgraph createSubgraph(const TestSubgraph& testSubgraph, uint32_t* constCopySize, + std::vector* constCopies, uint32_t* constRefSize, + std::vector* constReferences) { + CHECK(constCopySize != nullptr); + CHECK(constCopies != nullptr); + CHECK(constRefSize != nullptr); + CHECK(constReferences != nullptr); + + // Operands. + std::vector operands(testSubgraph.operands.size()); + for (uint32_t i = 0; i < testSubgraph.operands.size(); i++) { + const auto& op = testSubgraph.operands[i]; + + DataLocation loc = {}; + if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) { + loc = { + .poolIndex = 0, + .offset = *constCopySize, + .length = static_cast(op.data.size()), + }; + constCopies->push_back(&op.data); + *constCopySize += op.data.alignedSize(); + } else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) { + loc = { + .poolIndex = 0, + .offset = *constRefSize, + .length = static_cast(op.data.size()), + }; + constReferences->push_back(&op.data); + *constRefSize += op.data.alignedSize(); + } else if (op.lifetime == TestOperandLifeTime::SUBGRAPH) { + loc = { + .poolIndex = 0, + .offset = *op.data.get(), + .length = 0, + }; + } + + std::optional extraParams; + if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) { + using Tag = OperandExtraParams::Tag; + extraParams = OperandExtraParams::make(SymmPerChannelQuantParams{ + .scales = op.channelQuant.scales, + .channelDim = static_cast(op.channelQuant.channelDim)}); + } + + operands[i] = {.type = static_cast(op.type), + .dimensions = utils::toSigned(op.dimensions).value(), + .scale = op.scale, + .zeroPoint = op.zeroPoint, + .lifetime = static_cast(op.lifetime), + .location = loc, + .extraParams = std::move(extraParams)}; + } + + // Operations. + std::vector operations(testSubgraph.operations.size()); + std::transform(testSubgraph.operations.begin(), testSubgraph.operations.end(), + operations.begin(), [](const TestOperation& op) -> Operation { + return {.type = static_cast(op.type), + .inputs = utils::toSigned(op.inputs).value(), + .outputs = utils::toSigned(op.outputs).value()}; + }); + + return {.operands = std::move(operands), + .operations = std::move(operations), + .inputIndexes = utils::toSigned(testSubgraph.inputIndexes).value(), + .outputIndexes = utils::toSigned(testSubgraph.outputIndexes).value()}; +} + +void copyTestBuffers(const std::vector& buffers, uint8_t* output) { + uint32_t offset = 0; + for (const TestBuffer* buffer : buffers) { + const uint8_t* begin = buffer->get(); + const uint8_t* end = begin + buffer->size(); + std::copy(begin, end, output + offset); + offset += buffer->alignedSize(); + } +} + +} // namespace + +void waitForSyncFence(int syncFd) { + constexpr int kInfiniteTimeout = -1; + ASSERT_GT(syncFd, 0); + int r = sync_wait(syncFd, kInfiniteTimeout); + ASSERT_GE(r, 0); +} + +Model createModel(const TestModel& testModel) { + uint32_t constCopySize = 0; + uint32_t constRefSize = 0; + std::vector constCopies; + std::vector constReferences; + + Subgraph mainSubgraph = createSubgraph(testModel.main, &constCopySize, &constCopies, + &constRefSize, &constReferences); + std::vector refSubgraphs(testModel.referenced.size()); + std::transform(testModel.referenced.begin(), testModel.referenced.end(), refSubgraphs.begin(), + [&constCopySize, &constCopies, &constRefSize, + &constReferences](const TestSubgraph& testSubgraph) { + return createSubgraph(testSubgraph, &constCopySize, &constCopies, + &constRefSize, &constReferences); + }); + + // Constant copies. + std::vector operandValues(constCopySize); + copyTestBuffers(constCopies, operandValues.data()); + + // Shared memory. + std::vector pools = {}; + if (constRefSize > 0) { + const auto pool = nn::createSharedMemory(constRefSize).value(); + pools.push_back(pool); + + // load data + const auto mappedMemory = nn::map(pool).value(); + uint8_t* mappedPtr = static_cast(std::get(mappedMemory.pointer)); + CHECK(mappedPtr != nullptr); + + copyTestBuffers(constReferences, mappedPtr); + } + + std::vector aidlPools; + aidlPools.reserve(pools.size()); + for (auto& pool : pools) { + auto aidlPool = utils::convert(pool).value(); + aidlPools.push_back(std::move(aidlPool)); + } + + return {.main = std::move(mainSubgraph), + .referenced = std::move(refSubgraphs), + .operandValues = std::move(operandValues), + .pools = std::move(aidlPools), + .relaxComputationFloat32toFloat16 = testModel.isRelaxed}; +} + +static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) { + const auto byteSize = testModel.main.operands[testModel.main.outputIndexes[index]].data.size(); + return byteSize > 1u; +} + +static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) { + auto& length = request->outputs[outputIndex].location.length; + ASSERT_GT(length, 1u); + length -= 1u; +} + +static void makeOutputDimensionsUnspecified(Model* model) { + for (auto i : model->main.outputIndexes) { + auto& dims = model->main.operands[i].dimensions; + std::fill(dims.begin(), dims.end(), 0); + } +} + +// Manages the lifetime of memory resources used in an execution. +class ExecutionContext { + public: + ExecutionContext(std::shared_ptr device, std::shared_ptr preparedModel) + : kDevice(std::move(device)), kPreparedModel(std::move(preparedModel)) {} + + std::optional createRequest(const TestModel& testModel, MemoryType memoryType); + std::vector getOutputBuffers(const TestModel& testModel, + const Request& request) const; + + private: + // Get a TestBuffer with data copied from an IBuffer object. + void getBuffer(const std::shared_ptr& buffer, size_t size, + TestBuffer* testBuffer) const; + + static constexpr uint32_t kInputPoolIndex = 0; + static constexpr uint32_t kOutputPoolIndex = 1; + static constexpr uint32_t kDeviceMemoryBeginIndex = 2; + + const std::shared_ptr kDevice; + const std::shared_ptr kPreparedModel; + std::unique_ptr mInputMemory, mOutputMemory; + std::vector> mBuffers; +}; + +std::optional ExecutionContext::createRequest(const TestModel& testModel, + MemoryType memoryType) { + // Memory pools are organized as: + // - 0: Input shared memory pool + // - 1: Output shared memory pool + // - [2, 2+i): Input device memories + // - [2+i, 2+i+o): Output device memories + DeviceMemoryAllocator allocator(kDevice, kPreparedModel, testModel); + std::vector tokens; + mBuffers.clear(); + + // Model inputs. + std::vector inputs(testModel.main.inputIndexes.size()); + size_t inputSize = 0; + for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { + const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; + if (op.data.size() == 0) { + // Omitted input. + inputs[i] = {.hasNoValue = true}; + continue; + } else if (memoryType == MemoryType::DEVICE) { + SCOPED_TRACE("Input index = " + std::to_string(i)); + auto [buffer, token] = allocator.allocate(i); + if (buffer != nullptr) { + DataLocation loc = {.poolIndex = static_cast(mBuffers.size() + + kDeviceMemoryBeginIndex)}; + mBuffers.push_back(std::move(buffer)); + tokens.push_back(token); + inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + continue; + } + } + + // Reserve shared memory for input. + DataLocation loc = {.poolIndex = kInputPoolIndex, + .offset = static_cast(inputSize), + .length = static_cast(op.data.size())}; + inputSize += op.data.alignedSize(); + inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + } + + // Model outputs. + std::vector outputs(testModel.main.outputIndexes.size()); + size_t outputSize = 0; + for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) { + const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; + if (memoryType == MemoryType::DEVICE) { + SCOPED_TRACE("Output index = " + std::to_string(i)); + auto [buffer, token] = allocator.allocate(i); + if (buffer != nullptr) { + DataLocation loc = {.poolIndex = static_cast(mBuffers.size() + + kDeviceMemoryBeginIndex)}; + mBuffers.push_back(std::move(buffer)); + tokens.push_back(token); + outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + continue; + } + } + + // In the case of zero-sized output, we should at least provide a one-byte buffer. + // This is because zero-sized tensors are only supported internally to the driver, or + // reported in output shapes. It is illegal for the client to pre-specify a zero-sized + // tensor as model output. Otherwise, we will have two semantic conflicts: + // - "Zero dimension" conflicts with "unspecified dimension". + // - "Omitted operand buffer" conflicts with "zero-sized operand buffer". + size_t bufferSize = std::max(op.data.size(), 1); + + // Reserve shared memory for output. + DataLocation loc = {.poolIndex = kOutputPoolIndex, + .offset = static_cast(outputSize), + .length = static_cast(bufferSize)}; + outputSize += op.data.size() == 0 ? TestBuffer::kAlignment : op.data.alignedSize(); + outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + } + + if (memoryType == MemoryType::DEVICE && mBuffers.empty()) { + return std::nullopt; + } + + // Memory pools. + if (memoryType == MemoryType::BLOB_AHWB) { + mInputMemory = TestBlobAHWB::create(std::max(inputSize, 1)); + mOutputMemory = TestBlobAHWB::create(std::max(outputSize, 1)); + } else { + mInputMemory = TestAshmem::create(std::max(inputSize, 1)); + mOutputMemory = TestAshmem::create(std::max(outputSize, 1)); + } + CHECK_NE(mInputMemory, nullptr); + CHECK_NE(mOutputMemory, nullptr); + std::vector pools; + pools.reserve(kDeviceMemoryBeginIndex + mBuffers.size()); + + auto copiedInputMemory = utils::clone(*mInputMemory->getAidlMemory()); + CHECK(copiedInputMemory.has_value()) << copiedInputMemory.error().message; + auto copiedOutputMemory = utils::clone(*mOutputMemory->getAidlMemory()); + CHECK(copiedOutputMemory.has_value()) << copiedOutputMemory.error().message; + + pools.push_back(RequestMemoryPool::make( + std::move(copiedInputMemory).value())); + pools.push_back(RequestMemoryPool::make( + std::move(copiedOutputMemory).value())); + for (const auto& token : tokens) { + pools.push_back(RequestMemoryPool::make(token)); + } + + // Copy input data to the input shared memory pool. + uint8_t* inputPtr = mInputMemory->getPointer(); + for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { + if (!inputs[i].hasNoValue && inputs[i].location.poolIndex == kInputPoolIndex) { + const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; + const uint8_t* begin = op.data.get(); + const uint8_t* end = begin + op.data.size(); + std::copy(begin, end, inputPtr + inputs[i].location.offset); + } + } + return Request{ + .inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)}; +} + +std::vector ExecutionContext::getOutputBuffers(const TestModel& testModel, + const Request& request) const { + // Copy out output results. + uint8_t* outputPtr = mOutputMemory->getPointer(); + std::vector outputBuffers; + for (uint32_t i = 0; i < request.outputs.size(); i++) { + const auto& outputLoc = request.outputs[i].location; + if (outputLoc.poolIndex == kOutputPoolIndex) { + outputBuffers.emplace_back(outputLoc.length, outputPtr + outputLoc.offset); + } else { + const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; + if (op.data.size() == 0) { + outputBuffers.emplace_back(0, nullptr); + } else { + SCOPED_TRACE("Output index = " + std::to_string(i)); + const uint32_t bufferIndex = outputLoc.poolIndex - kDeviceMemoryBeginIndex; + TestBuffer buffer; + getBuffer(mBuffers[bufferIndex], op.data.size(), &buffer); + outputBuffers.push_back(std::move(buffer)); + } + } + } + return outputBuffers; +} + +// Get a TestBuffer with data copied from an IBuffer object. +void ExecutionContext::getBuffer(const std::shared_ptr& buffer, size_t size, + TestBuffer* testBuffer) const { + // IBuffer -> Shared memory. + auto sharedMemory = nn::createSharedMemory(size).value(); + auto aidlMemory = utils::convert(sharedMemory).value(); + const auto ret = buffer->copyTo(aidlMemory); + ASSERT_TRUE(ret.isOk()); + + // Shared memory -> TestBuffer. + const auto outputMemory = nn::map(sharedMemory).value(); + const uint8_t* outputPtr = std::visit( + [](auto* ptr) { return static_cast(ptr); }, outputMemory.pointer); + ASSERT_NE(outputPtr, nullptr); + ASSERT_NE(testBuffer, nullptr); + *testBuffer = TestBuffer(size, outputPtr); +} + +static bool hasZeroSizedOutput(const TestModel& testModel) { + return std::any_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(), + [&testModel](uint32_t index) { + return testModel.main.operands[index].data.size() == 0; + }); +} + +void EvaluatePreparedModel(const std::shared_ptr& device, + const std::shared_ptr& preparedModel, + const TestModel& testModel, const TestConfig& testConfig, + bool* skipped = nullptr) { + if (skipped != nullptr) { + *skipped = false; + } + // If output0 does not have size larger than one byte, we can not test with insufficient buffer. + if (testConfig.outputType == OutputType::INSUFFICIENT && + !isOutputSizeGreaterThanOne(testModel, 0)) { + return; + } + + ExecutionContext context(device, preparedModel); + auto maybeRequest = context.createRequest(testModel, testConfig.memoryType); + // Skip if testing memory domain but no device memory has been allocated. + if (!maybeRequest.has_value()) { + return; + } + + Request request = std::move(maybeRequest).value(); + + constexpr uint32_t kInsufficientOutputIndex = 0; + if (testConfig.outputType == OutputType::INSUFFICIENT) { + makeOutputInsufficientSize(kInsufficientOutputIndex, &request); + } + + int64_t loopTimeoutDuration = kOmittedTimeoutDuration; + // OutputType::MISSED_DEADLINE is only used by + // TestKind::INTINITE_LOOP_TIMEOUT tests to verify that an infinite loop is + // aborted after a timeout. + if (testConfig.outputType == OutputType::MISSED_DEADLINE) { + // Override the default loop timeout duration with a small value to + // speed up test execution. + constexpr int64_t kMillisecond = 1'000'000; + loopTimeoutDuration = 1 * kMillisecond; + } + + ErrorStatus executionStatus; + std::vector outputShapes; + Timing timing = kNoTiming; + switch (testConfig.executor) { + case Executor::SYNC: { + SCOPED_TRACE("synchronous"); + + ExecutionResult executionResult; + // execute + const auto ret = preparedModel->executeSynchronously(request, testConfig.measureTiming, + kNoDeadline, loopTimeoutDuration, + &executionResult); + ASSERT_TRUE(ret.isOk() || ret.getExceptionCode() == EX_SERVICE_SPECIFIC) + << ret.getDescription(); + if (ret.isOk()) { + executionStatus = executionResult.outputSufficientSize + ? ErrorStatus::NONE + : ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; + outputShapes = std::move(executionResult.outputShapes); + timing = executionResult.timing; + } else { + executionStatus = static_cast(ret.getServiceSpecificError()); + } + break; + } + case Executor::FENCED: { + SCOPED_TRACE("fenced"); + ErrorStatus result = ErrorStatus::NONE; + ndk::ScopedFileDescriptor syncFenceFd; + std::shared_ptr fencedCallback; + auto ret = preparedModel->executeFenced(request, {}, testConfig.measureTiming, + kNoDeadline, loopTimeoutDuration, kNoDuration, + &syncFenceFd, &fencedCallback); + ASSERT_TRUE(ret.isOk() || ret.getExceptionCode() == EX_SERVICE_SPECIFIC) + << ret.getDescription(); + if (!ret.isOk()) { + result = static_cast(ret.getServiceSpecificError()); + executionStatus = result; + } else if (syncFenceFd.get() != -1) { + std::vector waitFor; + auto dupFd = dup(syncFenceFd.get()); + ASSERT_NE(dupFd, -1); + waitFor.emplace_back(dupFd); + // If a sync fence is returned, try start another run waiting for the sync fence. + ret = preparedModel->executeFenced(request, waitFor, testConfig.measureTiming, + kNoDeadline, loopTimeoutDuration, kNoDuration, + &syncFenceFd, &fencedCallback); + ASSERT_TRUE(ret.isOk()); + waitForSyncFence(syncFenceFd.get()); + } + if (result == ErrorStatus::NONE) { + ASSERT_NE(fencedCallback, nullptr); + Timing timingFenced; + auto ret = + fencedCallback->getExecutionInfo(&timing, &timingFenced, &executionStatus); + ASSERT_TRUE(ret.isOk()); + } + break; + } + default: { + FAIL() << "Unsupported execution mode for AIDL interface."; + } + } + + if (testConfig.outputType != OutputType::FULLY_SPECIFIED && + executionStatus == ErrorStatus::GENERAL_FAILURE) { + if (skipped != nullptr) { + *skipped = true; + } + if (!testConfig.reportSkipping) { + return; + } + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "execute model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "execute model that it does not support." + << std::endl; + GTEST_SKIP(); + } + if (!testConfig.measureTiming) { + EXPECT_EQ(timing, kNoTiming); + } else { + if (timing.timeOnDevice != -1 && timing.timeInDriver != -1) { + EXPECT_LE(timing.timeOnDevice, timing.timeInDriver); + } + } + + switch (testConfig.outputType) { + case OutputType::FULLY_SPECIFIED: + if (testConfig.executor == Executor::FENCED && hasZeroSizedOutput(testModel)) { + // Executor::FENCED does not support zero-sized output. + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus); + return; + } + // If the model output operands are fully specified, outputShapes must be either + // either empty, or have the same number of elements as the number of outputs. + ASSERT_EQ(ErrorStatus::NONE, executionStatus); + ASSERT_TRUE(outputShapes.size() == 0 || + outputShapes.size() == testModel.main.outputIndexes.size()); + break; + case OutputType::UNSPECIFIED: + if (testConfig.executor == Executor::FENCED) { + // For Executor::FENCED, the output shape must be fully specified. + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus); + return; + } + // If the model output operands are not fully specified, outputShapes must have + // the same number of elements as the number of outputs. + ASSERT_EQ(ErrorStatus::NONE, executionStatus); + ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size()); + break; + case OutputType::INSUFFICIENT: + if (testConfig.executor == Executor::FENCED) { + // For Executor::FENCED, the output shape must be fully specified. + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionStatus); + return; + } + ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus); + ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size()); + // Check that all returned output dimensions are at least as fully specified as the + // union of the information about the corresponding operand in the model and in the + // request. In this test, all model outputs have known rank with all dimensions + // unspecified, and no dimensional information is provided in the request. + for (uint32_t i = 0; i < outputShapes.size(); i++) { + ASSERT_EQ(outputShapes[i].isSufficient, i != kInsufficientOutputIndex); + const auto& actual = outputShapes[i].dimensions; + const auto& golden = + testModel.main.operands[testModel.main.outputIndexes[i]].dimensions; + ASSERT_EQ(actual.size(), golden.size()); + for (uint32_t j = 0; j < actual.size(); j++) { + if (actual[j] == 0) continue; + EXPECT_EQ(actual[j], golden[j]) << "index: " << j; + } + } + return; + case OutputType::MISSED_DEADLINE: + ASSERT_TRUE(executionStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT || + executionStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT) + << "executionStatus = " << executionStatus; + return; + } + + // Go through all outputs, check returned output shapes. + for (uint32_t i = 0; i < outputShapes.size(); i++) { + EXPECT_TRUE(outputShapes[i].isSufficient); + const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions; + const auto unsignedActual = nn::toUnsigned(outputShapes[i].dimensions); + ASSERT_TRUE(unsignedActual.has_value()); + const std::vector& actual = unsignedActual.value(); + EXPECT_EQ(expect, actual); + } + + // Retrieve execution results. + const std::vector outputs = context.getOutputBuffers(testModel, request); + + // We want "close-enough" results. + checkResults(testModel, outputs); +} + +void EvaluatePreparedModel(const std::shared_ptr& device, + const std::shared_ptr& preparedModel, + const TestModel& testModel, TestKind testKind) { + std::vector outputTypesList; + std::vector measureTimingList; + std::vector executorList; + std::vector memoryTypeList; + + switch (testKind) { + case TestKind::GENERAL: { + outputTypesList = {OutputType::FULLY_SPECIFIED}; + measureTimingList = {false, true}; + executorList = {Executor::SYNC}; + memoryTypeList = {MemoryType::ASHMEM}; + } break; + case TestKind::DYNAMIC_SHAPE: { + outputTypesList = {OutputType::UNSPECIFIED, OutputType::INSUFFICIENT}; + measureTimingList = {false, true}; + executorList = {Executor::SYNC, Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; + } break; + case TestKind::MEMORY_DOMAIN: { + outputTypesList = {OutputType::FULLY_SPECIFIED}; + measureTimingList = {false}; + executorList = {Executor::SYNC, Executor::FENCED}; + memoryTypeList = {MemoryType::BLOB_AHWB, MemoryType::DEVICE}; + } break; + case TestKind::FENCED_COMPUTE: { + outputTypesList = {OutputType::FULLY_SPECIFIED}; + measureTimingList = {false, true}; + executorList = {Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; + } break; + case TestKind::QUANTIZATION_COUPLING: { + LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel"; + return; + } break; + case TestKind::INTINITE_LOOP_TIMEOUT: { + outputTypesList = {OutputType::MISSED_DEADLINE}; + measureTimingList = {false, true}; + executorList = {Executor::SYNC, Executor::FENCED}; + memoryTypeList = {MemoryType::ASHMEM}; + } break; + } + + for (const OutputType outputType : outputTypesList) { + for (const bool measureTiming : measureTimingList) { + for (const Executor executor : executorList) { + for (const MemoryType memoryType : memoryTypeList) { + const TestConfig testConfig(executor, measureTiming, outputType, memoryType); + EvaluatePreparedModel(device, preparedModel, testModel, testConfig); + } + } + } + } +} + +void EvaluatePreparedCoupledModels(const std::shared_ptr& device, + const std::shared_ptr& preparedModel, + const TestModel& testModel, + const std::shared_ptr& preparedCoupledModel, + const TestModel& coupledModel) { + const std::vector outputTypesList = {OutputType::FULLY_SPECIFIED}; + const std::vector measureTimingList = {false, true}; + const std::vector executorList = {Executor::SYNC, Executor::FENCED}; + + for (const OutputType outputType : outputTypesList) { + for (const bool measureTiming : measureTimingList) { + for (const Executor executor : executorList) { + const TestConfig testConfig(executor, measureTiming, outputType, MemoryType::ASHMEM, + /*reportSkipping=*/false); + bool baseSkipped = false; + EvaluatePreparedModel(device, preparedModel, testModel, testConfig, &baseSkipped); + bool coupledSkipped = false; + EvaluatePreparedModel(device, preparedCoupledModel, coupledModel, testConfig, + &coupledSkipped); + ASSERT_EQ(baseSkipped, coupledSkipped); + if (baseSkipped) { + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "execute model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service " + "cannot " + "execute model that it does not support." + << std::endl; + GTEST_SKIP(); + } + } + } + } +} + +void Execute(const std::shared_ptr& device, const TestModel& testModel, + TestKind testKind) { + Model model = createModel(testModel); + if (testKind == TestKind::DYNAMIC_SHAPE) { + makeOutputDimensionsUnspecified(&model); + } + + std::shared_ptr preparedModel; + switch (testKind) { + case TestKind::GENERAL: + case TestKind::DYNAMIC_SHAPE: + case TestKind::MEMORY_DOMAIN: + case TestKind::FENCED_COMPUTE: + case TestKind::INTINITE_LOOP_TIMEOUT: { + createPreparedModel(device, model, &preparedModel); + if (preparedModel == nullptr) return; + EvaluatePreparedModel(device, preparedModel, testModel, testKind); + } break; + case TestKind::QUANTIZATION_COUPLING: { + ASSERT_TRUE(testModel.hasQuant8CoupledOperands()); + createPreparedModel(device, model, &preparedModel, + /*reportSkipping*/ false); + TestModel signedQuantizedModel = convertQuant8AsymmOperandsToSigned(testModel); + std::shared_ptr preparedCoupledModel; + createPreparedModel(device, createModel(signedQuantizedModel), &preparedCoupledModel, + /*reportSkipping*/ false); + // If we couldn't prepare a model with unsigned quantization, we must + // fail to prepare a model with signed quantization as well. + if (preparedModel == nullptr) { + ASSERT_EQ(preparedCoupledModel, nullptr); + // If we failed to prepare both of the models, we can safely skip + // the test. + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " + "prepare model that it does not support."; + std::cout + << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + GTEST_SKIP(); + } + ASSERT_NE(preparedCoupledModel, nullptr); + EvaluatePreparedCoupledModels(device, preparedModel, testModel, preparedCoupledModel, + signedQuantizedModel); + } break; + } +} + +void GeneratedTestBase::SetUp() { + testing::TestWithParam::SetUp(); + ASSERT_NE(kDevice, nullptr); +} + +std::vector getNamedModels(const FilterFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + +std::vector getNamedModels(const FilterNameFn& filter) { + return TestModelManager::get().getTestModels(filter); +} + +std::string printGeneratedTest(const testing::TestParamInfo& info) { + const auto& [namedDevice, namedModel] = info.param; + return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel)); +} + +// Tag for the generated tests +class GeneratedTest : public GeneratedTestBase {}; + +// Tag for the dynamic output shape tests +class DynamicOutputShapeTest : public GeneratedTest {}; + +// Tag for the memory domain tests +class MemoryDomainTest : public GeneratedTest {}; + +// Tag for the fenced compute tests +class FencedComputeTest : public GeneratedTest {}; + +// Tag for the dynamic output shape tests +class QuantizationCouplingTest : public GeneratedTest {}; + +// Tag for the loop timeout tests +class InfiniteLoopTimeoutTest : public GeneratedTest {}; + +TEST_P(GeneratedTest, Test) { + Execute(kDevice, kTestModel, TestKind::GENERAL); +} + +TEST_P(DynamicOutputShapeTest, Test) { + Execute(kDevice, kTestModel, TestKind::DYNAMIC_SHAPE); +} + +TEST_P(MemoryDomainTest, Test) { + Execute(kDevice, kTestModel, TestKind::MEMORY_DOMAIN); +} + +TEST_P(FencedComputeTest, Test) { + Execute(kDevice, kTestModel, TestKind::FENCED_COMPUTE); +} + +TEST_P(QuantizationCouplingTest, Test) { + Execute(kDevice, kTestModel, TestKind::QUANTIZATION_COUPLING); +} + +TEST_P(InfiniteLoopTimeoutTest, Test) { + Execute(kDevice, kTestModel, TestKind::INTINITE_LOOP_TIMEOUT); +} + +INSTANTIATE_GENERATED_TEST(GeneratedTest, + [](const TestModel& testModel) { return !testModel.expectFailure; }); + +INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest, [](const TestModel& testModel) { + return !testModel.expectFailure && !testModel.hasScalarOutputs(); +}); + +INSTANTIATE_GENERATED_TEST(MemoryDomainTest, + [](const TestModel& testModel) { return !testModel.expectFailure; }); + +INSTANTIATE_GENERATED_TEST(FencedComputeTest, + [](const TestModel& testModel) { return !testModel.expectFailure; }); + +INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testModel) { + return !testModel.expectFailure && testModel.hasQuant8CoupledOperands() && + testModel.main.operations.size() == 1; +}); + +INSTANTIATE_GENERATED_TEST(InfiniteLoopTimeoutTest, [](const TestModel& testModel) { + return testModel.isInfiniteLoopTimeoutTest(); +}); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.h b/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.h new file mode 100644 index 0000000000..ad40f06874 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/GeneratedTestHarness.h @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_NEURALNETWORKS_AIDL_GENERATED_TEST_HARNESS_H +#define ANDROID_HARDWARE_NEURALNETWORKS_AIDL_GENERATED_TEST_HARNESS_H + +#include +#include + +#include +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using NamedModel = Named; +using GeneratedTestParam = std::tuple; + +class GeneratedTestBase : public testing::TestWithParam { + protected: + void SetUp() override; + const std::shared_ptr kDevice = getData(std::get(GetParam())); + const test_helper::TestModel& kTestModel = *getData(std::get(GetParam())); +}; + +using FilterFn = std::function; +std::vector getNamedModels(const FilterFn& filter); + +using FilterNameFn = std::function; +std::vector getNamedModels(const FilterNameFn& filter); + +std::string printGeneratedTest(const testing::TestParamInfo& info); + +#define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \ + GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TestSuite); \ + INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite, \ + testing::Combine(testing::ValuesIn(getNamedDevices()), \ + testing::ValuesIn(getNamedModels(filter))), \ + printGeneratedTest) + +// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp. +// TODO: Clean up the hierarchy for ValidationTest. +class ValidationTest : public GeneratedTestBase {}; + +Model createModel(const test_helper::TestModel& testModel); + +void PrepareModel(const std::shared_ptr& device, const Model& model, + std::shared_ptr* preparedModel); + +enum class TestKind { + // Runs a test model and compares the results to a golden data + GENERAL, + // Same as GENERAL but sets dimensions for the output tensors to zeros + DYNAMIC_SHAPE, + // Same as GENERAL but use device memories for inputs and outputs + MEMORY_DOMAIN, + // Same as GENERAL but use executeFenced for exeuction + FENCED_COMPUTE, + // Tests if quantized model with TENSOR_QUANT8_ASYMM produces the same result + // (OK/SKIPPED/FAILED) as the model with all such tensors converted to + // TENSOR_QUANT8_ASYMM_SIGNED. + QUANTIZATION_COUPLING, + // Runs a test model and verifies that MISSED_DEADLINE_* is returned. + INTINITE_LOOP_TIMEOUT +}; + +void EvaluatePreparedModel(const std::shared_ptr& device, + const std::shared_ptr& preparedModel, + const test_helper::TestModel& testModel, TestKind testKind); + +void waitForSyncFence(int syncFd); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional + +#endif // ANDROID_HARDWARE_NEURALNETWORKS_AIDL_GENERATED_TEST_HARNESS_H diff --git a/neuralnetworks/aidl/vts/functional/LogTestCaseToLogcat.h b/neuralnetworks/aidl/vts/functional/LogTestCaseToLogcat.h new file mode 100644 index 0000000000..c9fd432a43 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/LogTestCaseToLogcat.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_NEURALNETWORKS_AIDL_LOG_TEST_CASE_TO_LOGCAT_H +#define ANDROID_HARDWARE_NEURALNETWORKS_AIDL_LOG_TEST_CASE_TO_LOGCAT_H + +#include +#include + +namespace aidl::android::hardware::neuralnetworks { + +class LogTestCaseToLogcat : public ::testing::EmptyTestEventListener { + public: + void OnTestStart(const ::testing::TestInfo& test_info) override { + LOG(INFO) << "[Test Case] " << test_info.test_suite_name() << "." << test_info.name() + << " BEGIN"; + } + + void OnTestEnd(const ::testing::TestInfo& test_info) override { + LOG(INFO) << "[Test Case] " << test_info.test_suite_name() << "." << test_info.name() + << " END"; + } +}; + +} // namespace aidl::android::hardware::neuralnetworks + +#endif // ANDROID_HARDWARE_NEURALNETWORKS_AIDL_LOG_TEST_CASE_TO_LOGCAT_H diff --git a/neuralnetworks/aidl/vts/functional/MemoryDomainTests.cpp b/neuralnetworks/aidl/vts/functional/MemoryDomainTests.cpp new file mode 100644 index 0000000000..a37a0caa29 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/MemoryDomainTests.cpp @@ -0,0 +1,1176 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "AidlHalInterfaces.h" +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "MemoryUtils.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using namespace test_helper; +using implementation::PreparedModelCallback; + +namespace { + +// An AIDL driver is likely to support at least one of the following operand types. +const std::vector kTestOperandTypeChoicesVector = { + TestOperandType::TENSOR_FLOAT32, + TestOperandType::TENSOR_FLOAT16, + TestOperandType::TENSOR_QUANT8_ASYMM, + TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, +}; +const auto kTestOperandTypeChoices = testing::ValuesIn(kTestOperandTypeChoicesVector); +// TODO(b/179270601): restore kNamedDeviceChoices + +bool isInChoices(TestOperandType type) { + return std::count(kTestOperandTypeChoicesVector.begin(), kTestOperandTypeChoicesVector.end(), + type) > 0; +} + +bool isFloat(TestOperandType type) { + CHECK(isInChoices(type)); + return type == TestOperandType::TENSOR_FLOAT32 || type == TestOperandType::TENSOR_FLOAT16; +} + +// Create placeholder buffers for model constants as well as inputs and outputs. +// We only care about the size here because we will not check accuracy in validation tests. +void createDummyData(TestModel* testModel) { + for (auto& operand : testModel->main.operands) { + if (operand.data != nullptr) continue; + switch (operand.lifetime) { + case TestOperandLifeTime::SUBGRAPH_INPUT: + case TestOperandLifeTime::SUBGRAPH_OUTPUT: + case TestOperandLifeTime::CONSTANT_COPY: + case TestOperandLifeTime::CONSTANT_REFERENCE: { + const uint32_t size = nn::nonExtensionOperandSizeOfData( + static_cast(operand.type), operand.dimensions); + operand.data = TestBuffer(size); + } break; + default: + break; + } + } +} + +TestOperand createInt32Scalar(int32_t value) { + return { + .type = TestOperandType::INT32, + .dimensions = {}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY, + .data = TestBuffer::createFromVector({value}), + }; +} + +// Construct a test model with multiple CONV_2D operations with the given operand as inputs. +// The dimensions of the filters are chosen to ensure outputs has the same dimensions as inputs. +// We choose CONV_2D operation because it is commonly supported by most drivers. +TestModel createConvModel(const TestOperand& operand, uint32_t numOperations) { + CHECK(isInChoices(operand.type)); + + TestOperand weight = {.type = operand.type, + .dimensions = {operand.dimensions[3], 3, 3, operand.dimensions[3]}, + .numberOfConsumers = 1, + .scale = isFloat(operand.type) ? 0.0f : 1.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY}; + + TestOperand bias = { + .type = isFloat(operand.type) ? operand.type : TestOperandType::TENSOR_INT32, + .dimensions = {operand.dimensions[3]}, + .numberOfConsumers = 1, + .scale = operand.scale * weight.scale, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::CONSTANT_COPY}; + + TestOperand output = operand; + output.numberOfConsumers = 0; + output.lifetime = TestOperandLifeTime::SUBGRAPH_OUTPUT; + + const std::vector operands = { + operand, + std::move(weight), + std::move(bias), + createInt32Scalar(1), // same padding + createInt32Scalar(1), // width stride + createInt32Scalar(1), // height stride + createInt32Scalar(0), // activation = NONE + std::move(output), + }; + + TestModel model; + for (uint32_t i = 0; i < numOperations; i++) { + model.main.operands.insert(model.main.operands.end(), operands.begin(), operands.end()); + const uint32_t inputIndex = operands.size() * i; + const uint32_t outputIndex = inputIndex + operands.size() - 1; + std::vector inputs(operands.size() - 1); + std::iota(inputs.begin(), inputs.end(), inputIndex); + model.main.operations.push_back({.type = TestOperationType::CONV_2D, + .inputs = std::move(inputs), + .outputs = {outputIndex}}); + model.main.inputIndexes.push_back(inputIndex); + model.main.outputIndexes.push_back(outputIndex); + } + createDummyData(&model); + return model; +} + +// Construct a test model with a single ADD operation with the given operand as input0 and input1. +// This is to cover additional cases that the CONV_2D model does not support, e.g. arbitrary input +// operand rank, scalar input operand. We choose ADD operation because it is commonly supported by +// most drivers. +TestModel createSingleAddModel(const TestOperand& operand) { + CHECK(isInChoices(operand.type)); + + TestOperand act = { + .type = TestOperandType::INT32, + .dimensions = {}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }; + + TestOperand output = operand; + output.numberOfConsumers = 0; + output.lifetime = TestOperandLifeTime::SUBGRAPH_OUTPUT; + + TestModel model = { + .main = + { + .operands = + { + operand, + operand, + std::move(act), + output, + }, + .operations = {{.type = TestOperationType::ADD, + .inputs = {0, 1, 2}, + .outputs = {3}}}, + .inputIndexes = {0, 1, 2}, + .outputIndexes = {3}, + }, + }; + createDummyData(&model); + return model; +} + +// A placeholder invalid IPreparedModel class for MemoryDomainAllocateTest.InvalidPreparedModel +class InvalidPreparedModel : public BnPreparedModel { + public: + ndk::ScopedAStatus executeSynchronously(const Request&, bool, int64_t, int64_t, + ExecutionResult*) override { + return ndk::ScopedAStatus::fromServiceSpecificError( + static_cast(ErrorStatus::GENERAL_FAILURE)); + } + ndk::ScopedAStatus executeFenced(const Request&, const std::vector&, + bool, int64_t, int64_t, int64_t, ndk::ScopedFileDescriptor*, + std::shared_ptr*) override { + return ndk::ScopedAStatus::fromServiceSpecificError( + static_cast(ErrorStatus::GENERAL_FAILURE)); + } +}; + +template +std::vector createRequestMemoryPools(const Args&... pools) { + std::vector memoryPools; + memoryPools.reserve(sizeof...(Args)); + // This fold operator calls push_back on each of the function arguments. + (memoryPools.push_back(utils::clone(pools).value()), ...); + return memoryPools; +}; + +} // namespace + +class MemoryDomainTestBase : public testing::Test { + protected: + MemoryDomainTestBase(std::shared_ptr device, TestOperandType type) + : kDevice(std::move(device)), + kTestOperandType(type), + kTestOperand(kTestOperandMap.at(type)), + kTestOperandDataSize(nn::nonExtensionOperandSizeOfData(static_cast(type), + kTestOperand.dimensions)) {} + + void SetUp() override { + testing::Test::SetUp(); + ASSERT_NE(kDevice, nullptr); + } + + std::shared_ptr createConvPreparedModel(const TestOperand& testOperand, + uint32_t numOperations = 1) { + const TestModel testModel = createConvModel(testOperand, numOperations); + const Model model = createModel(testModel); + std::shared_ptr preparedModel; + createPreparedModel(kDevice, model, &preparedModel, /*reportSkipping=*/false); + return preparedModel; + } + + std::shared_ptr createAddPreparedModel(const TestOperand& testOperand) { + const TestModel testModel = createSingleAddModel(testOperand); + const Model model = createModel(testModel); + std::shared_ptr preparedModel; + createPreparedModel(kDevice, model, &preparedModel, /*reportSkipping=*/false); + return preparedModel; + } + + static const std::map kTestOperandMap; + + const std::shared_ptr kDevice; + const TestOperandType kTestOperandType; + const TestOperand& kTestOperand; + const uint32_t kTestOperandDataSize; +}; + +const std::map MemoryDomainTestBase::kTestOperandMap = { + {TestOperandType::TENSOR_FLOAT32, + { + .type = TestOperandType::TENSOR_FLOAT32, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_FLOAT16, + { + .type = TestOperandType::TENSOR_FLOAT16, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_QUANT8_ASYMM, + { + .type = TestOperandType::TENSOR_QUANT8_ASYMM, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.5f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, + {TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, + { + .type = TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, + .dimensions = {1, 32, 32, 8}, + .numberOfConsumers = 1, + .scale = 0.5f, + .zeroPoint = 0, + .lifetime = TestOperandLifeTime::SUBGRAPH_INPUT, + }}, +}; + +using MemoryDomainAllocateTestParam = std::tuple; +class MemoryDomainAllocateTest : public MemoryDomainTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainAllocateTest() + : MemoryDomainTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} + + struct AllocateTestArgs { + std::vector dimensions; + std::vector> preparedModels; + std::vector inputRoles; + std::vector outputRoles; + }; + + // Validation test for IDevice::allocate. The driver is expected to fail with INVALID_ARGUMENT, + // or GENERAL_FAILURE if memory domain is not supported. + void validateAllocate(AllocateTestArgs args) { + std::vector preparedModelParcels; + preparedModelParcels.reserve(args.preparedModels.size()); + for (const auto& model : args.preparedModels) { + preparedModelParcels.push_back({.preparedModel = model}); + } + DeviceBuffer buffer; + const auto ret = + kDevice->allocate({.dimensions = std::move(args.dimensions)}, preparedModelParcels, + args.inputRoles, args.outputRoles, &buffer); + + ASSERT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_TRUE(static_cast(ret.getServiceSpecificError()) == + ErrorStatus::INVALID_ARGUMENT || + static_cast(ret.getServiceSpecificError()) == + ErrorStatus::GENERAL_FAILURE); + } + + void testConflictOperands(const std::shared_ptr& model1, + const std::shared_ptr& model2) { + validateAllocate({ + .preparedModels = {model1, model2}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {model1, model2}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + .outputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {model1, model2}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + } +}; + +TEST_P(MemoryDomainAllocateTest, EmptyRole) { + // Test with empty prepared models and roles. + validateAllocate({}); + + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // Test again with non-empty prepared models but empty roles. + validateAllocate({ + .preparedModels = {preparedModel}, + }); +} + +TEST_P(MemoryDomainAllocateTest, NullptrPreparedModel) { + // Test with nullptr prepared model as input role. + validateAllocate({ + .preparedModels = {nullptr}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Test with nullptr prepared model as output role. + validateAllocate({ + .preparedModels = {nullptr}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidPreparedModel) { + std::shared_ptr invalidPreparedModel = + ndk::SharedRefBase::make(); + + // Test with invalid prepared model as input role. + validateAllocate({ + .preparedModels = {invalidPreparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Test with invalid prepared model as output role. + validateAllocate({ + .preparedModels = {invalidPreparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidModelIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the model index is out of bound. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // This should fail, because the model index is out of bound. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidIOIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the model only has one input. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 1, .frequency = 1.0f}}, + }); + + // This should fail, because the model only has one output. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 1, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, InvalidFrequency) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + for (float invalidFreq : {10.0f, 0.0f, -0.5f}) { + // Test with invalid frequency for input roles. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = invalidFreq}}, + }); + // Test with invalid frequency for output roles. + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = invalidFreq}}, + }); + } +} + +TEST_P(MemoryDomainAllocateTest, SameRoleSpecifiedTwice) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // Same role with same model index. + validateAllocate({ + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + + // Different model indexes, but logically referring to the same role. + validateAllocate({ + .preparedModels = {preparedModel, preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .preparedModels = {preparedModel, preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}, + {.modelIndex = 1, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictOperandType) { + const std::map conflictTypeMap = { + {TestOperandType::TENSOR_FLOAT32, TestOperandType::TENSOR_FLOAT16}, + {TestOperandType::TENSOR_FLOAT16, TestOperandType::TENSOR_FLOAT32}, + {TestOperandType::TENSOR_QUANT8_ASYMM, TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED}, + {TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED, TestOperandType::TENSOR_QUANT8_ASYMM}, + }; + + TestOperand conflictTestOperand = kTestOperand; + const auto it = conflictTypeMap.find(kTestOperandType); + ASSERT_FALSE(it == conflictTypeMap.end()); + conflictTestOperand.type = it->second; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictScale) { + if (isFloat(kTestOperandType)) return; + + TestOperand conflictTestOperand = kTestOperand; + ASSERT_NE(conflictTestOperand.scale, 1.0f); + conflictTestOperand.scale = 1.0f; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictZeroPoint) { + if (isFloat(kTestOperandType)) return; + + TestOperand conflictTestOperand = kTestOperand; + ASSERT_NE(conflictTestOperand.zeroPoint, 10); + conflictTestOperand.zeroPoint = 10; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankBetweenRoles) { + TestOperand conflictTestOperand = kTestOperand; + conflictTestOperand.dimensions.pop_back(); + + auto preparedModel = createAddPreparedModel(kTestOperand); + auto conflictPreparedModel = createAddPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictDimensionsBetweenRoles) { + TestOperand conflictTestOperand = kTestOperand; + conflictTestOperand.dimensions[0] = 4; + + auto preparedModel = createConvPreparedModel(kTestOperand); + auto conflictPreparedModel = createConvPreparedModel(conflictTestOperand); + if (preparedModel == nullptr || conflictPreparedModel == nullptr) return; + testConflictOperands(preparedModel, conflictPreparedModel); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankBetweenRoleAndDesc) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + auto badDimensions = utils::toSigned(kTestOperand.dimensions).value(); + badDimensions.pop_back(); + + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictDimensionsBetweenRoleAndDesc) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + auto badDimensions = utils::toSigned(kTestOperand.dimensions).value(); + badDimensions[0] = 4; + + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); + validateAllocate({ + .dimensions = badDimensions, + .preparedModels = {preparedModel}, + .outputRoles = {{.modelIndex = 0, .ioIndex = 0, .frequency = 1.0f}}, + }); +} + +TEST_P(MemoryDomainAllocateTest, ConflictRankWithScalarRole) { + auto preparedModel = createAddPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + // This should fail, because the target operand is a scalar but a non-empty dimension is + // specified. + validateAllocate({ + .dimensions = {1}, + .preparedModels = {preparedModel}, + .inputRoles = {{.modelIndex = 0, .ioIndex = 2, .frequency = 1.0f}}, + }); +} + +std::string printMemoryDomainAllocateTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType] = info.param; + const std::string type = toString(static_cast(operandType)); + return gtestCompliantName(getName(namedDevice) + "_" + type); +} + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(MemoryDomainAllocateTest); +INSTANTIATE_TEST_SUITE_P(TestMemoryDomain, MemoryDomainAllocateTest, + testing::Combine(testing::ValuesIn(getNamedDevices()), + kTestOperandTypeChoices), + printMemoryDomainAllocateTest); + +class MemoryDomainCopyTestBase : public MemoryDomainTestBase { + protected: + MemoryDomainCopyTestBase(std::shared_ptr device, TestOperandType type) + : MemoryDomainTestBase(std::move(device), type) {} + + // Allocates device memory for roles of a single prepared model. + // Returns {IBuffer, token} if success; returns {nullptr, 0} if not supported. + DeviceBuffer allocateBuffer(const std::shared_ptr& preparedModel, + const std::vector& inputIndexes, + const std::vector& outputIndexes, + const std::vector& dimensions) { + if (preparedModel == nullptr) { + return {.buffer = nullptr, .token = 0}; + } + + std::vector inputRoles(inputIndexes.size()), outputRoles(outputIndexes.size()); + auto trans = [](int32_t ind) -> BufferRole { + return {.modelIndex = 0, .ioIndex = ind, .frequency = 1.0f}; + }; + std::transform(inputIndexes.begin(), inputIndexes.end(), inputRoles.begin(), trans); + std::transform(outputIndexes.begin(), outputIndexes.end(), outputRoles.begin(), trans); + + IPreparedModelParcel parcel; + parcel.preparedModel = preparedModel; + + DeviceBuffer buffer; + + const auto ret = kDevice->allocate({.dimensions = dimensions}, {parcel}, inputRoles, + outputRoles, &buffer); + + if (!ret.isOk()) { + EXPECT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + EXPECT_EQ(static_cast(ret.getServiceSpecificError()), + ErrorStatus::GENERAL_FAILURE); + return DeviceBuffer{ + .buffer = nullptr, + .token = 0, + }; + } + + EXPECT_NE(buffer.buffer, nullptr); + EXPECT_GT(buffer.token, 0); + + return buffer; + } + + DeviceBuffer allocateBuffer(const std::shared_ptr& preparedModel, + const std::vector& inputIndexes, + const std::vector& outputIndexes) { + return allocateBuffer(preparedModel, inputIndexes, outputIndexes, {}); + } + + Memory allocateSharedMemory(uint32_t size) { + const auto sharedMemory = nn::createSharedMemory(size).value(); + auto memory = utils::convert(sharedMemory).value(); + EXPECT_EQ(memory.size, size); + return memory; + } + + void testCopyFrom(const std::shared_ptr& buffer, const Memory& memory, + const std::vector& dimensions, ErrorStatus expectedStatus) { + const auto ret = buffer->copyFrom(memory, dimensions); + if (expectedStatus == ErrorStatus::NONE) { + ASSERT_TRUE(ret.isOk()); + } else { + ASSERT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(expectedStatus, static_cast(ret.getServiceSpecificError())); + } + } + + void testCopyTo(const std::shared_ptr& buffer, const Memory& memory, + ErrorStatus expectedStatus) { + const auto ret = buffer->copyTo(memory); + if (expectedStatus == ErrorStatus::NONE) { + ASSERT_TRUE(ret.isOk()); + } else { + ASSERT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(expectedStatus, static_cast(ret.getServiceSpecificError())); + } + } + + void initializeDeviceMemory(const std::shared_ptr& buffer) { + Memory memory = allocateSharedMemory(kTestOperandDataSize); + ASSERT_EQ(memory.size, kTestOperandDataSize); + testCopyFrom(buffer, memory, utils::toSigned(kTestOperand.dimensions).value(), + ErrorStatus::NONE); + } +}; + +using MemoryDomainCopyTestParam = std::tuple; +class MemoryDomainCopyTest : public MemoryDomainCopyTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainCopyTest() + : MemoryDomainCopyTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} +}; + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidMemorySize) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + Memory badMemory1 = allocateSharedMemory(badMemorySize1); + Memory badMemory2 = allocateSharedMemory(badMemorySize2); + testCopyFrom(buffer, badMemory1, {}, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, badMemory2, {}, ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidMemorySize_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + Memory badMemory1 = allocateSharedMemory(badMemorySize1); + Memory badMemory2 = allocateSharedMemory(badMemorySize2); + Memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + const auto goodDimensions = utils::toSigned(kTestOperand.dimensions).value(); + auto badDimensions = goodDimensions; + badDimensions[0] = 2; + + testCopyFrom(buffer, badMemory1, goodDimensions, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, badMemory2, goodDimensions, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, goodMemory, goodDimensions, ErrorStatus::NONE); + testCopyFrom(buffer, goodMemory, badDimensions, ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidDimensions) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + Memory memory = allocateSharedMemory(kTestOperandDataSize); + + const auto goodDimensions = utils::toSigned(kTestOperand.dimensions).value(); + std::vector badDimensions = goodDimensions; + badDimensions.pop_back(); + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = goodDimensions; + badDimensions[0] = 2; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = goodDimensions; + badDimensions[0] = 0; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + testCopyFrom(buffer, memory, {}, ErrorStatus::NONE); + testCopyFrom(buffer, memory, goodDimensions, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyFrom_InvalidDimensions_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + Memory memory = allocateSharedMemory(kTestOperandDataSize); + + const auto goodDimensions = utils::toSigned(kTestOperand.dimensions).value(); + std::vector badDimensions = goodDimensions; + badDimensions.pop_back(); + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = goodDimensions; + badDimensions[0] = 2; + badDimensions[3] = 4; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + badDimensions = goodDimensions; + badDimensions[0] = 1; + badDimensions[3] = 0; + testCopyFrom(buffer, memory, badDimensions, ErrorStatus::INVALID_ARGUMENT); + + testCopyFrom(buffer, memory, {}, ErrorStatus::INVALID_ARGUMENT); + testCopyFrom(buffer, memory, goodDimensions, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_UninitializedMemory) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + Memory memory = allocateSharedMemory(kTestOperandDataSize); + testCopyTo(buffer, memory, ErrorStatus::GENERAL_FAILURE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_InvalidMemorySize) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + Memory badMemory1 = allocateSharedMemory(badMemorySize1); + Memory badMemory2 = allocateSharedMemory(badMemorySize2); + Memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + initializeDeviceMemory(buffer); + testCopyTo(buffer, badMemory1, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, badMemory2, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, goodMemory, ErrorStatus::NONE); +} + +TEST_P(MemoryDomainCopyTest, CopyTo_InvalidMemorySize_DynamicShape) { + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + uint32_t badMemorySize1 = kTestOperandDataSize / 2, badMemorySize2 = kTestOperandDataSize * 2; + Memory badMemory1 = allocateSharedMemory(badMemorySize1); + Memory badMemory2 = allocateSharedMemory(badMemorySize2); + Memory goodMemory = allocateSharedMemory(kTestOperandDataSize); + + initializeDeviceMemory(buffer); + testCopyTo(buffer, badMemory1, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, badMemory2, ErrorStatus::INVALID_ARGUMENT); + testCopyTo(buffer, goodMemory, ErrorStatus::NONE); +} + +std::string printMemoryDomainCopyTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType] = info.param; + const std::string type = toString(static_cast(operandType)); + return gtestCompliantName(getName(namedDevice) + "_" + type); +} + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(MemoryDomainCopyTest); +INSTANTIATE_TEST_SUITE_P(TestMemoryDomain, MemoryDomainCopyTest, + testing::Combine(testing::ValuesIn(getNamedDevices()), + kTestOperandTypeChoices), + printMemoryDomainCopyTest); + +using MemoryDomainExecutionTestParam = std::tuple; +class MemoryDomainExecutionTest + : public MemoryDomainCopyTestBase, + public testing::WithParamInterface { + protected: + MemoryDomainExecutionTest() + : MemoryDomainCopyTestBase(getData(std::get(GetParam())), + std::get(GetParam())) {} + + RequestMemoryPool createSharedMemoryPool(uint32_t size) { + return RequestMemoryPool(allocateSharedMemory(size)); + } + + RequestMemoryPool createDeviceMemoryPool(uint32_t token) { + return RequestMemoryPool(static_cast(token)); + } + + void testExecution(const std::shared_ptr& preparedModel, const Request& request, + ErrorStatus expectedStatus) { + switch (kExecutor) { + case Executor::SYNC: + EXPECT_EQ(executeSync(preparedModel, request), expectedStatus); + break; + case Executor::FENCED: + EXPECT_EQ(executeFenced(preparedModel, request), expectedStatus); + break; + default: + ASSERT_TRUE(false); + } + } + + ErrorStatus executeSync(const std::shared_ptr& preparedModel, + const Request& request) { + ExecutionResult executionResult; + const auto ret = preparedModel->executeSynchronously( + request, false, kNoDeadline, kOmittedTimeoutDuration, &executionResult); + + if (!ret.isOk()) { + EXPECT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + return static_cast(ret.getServiceSpecificError()); + } + const ErrorStatus executionStatus = executionResult.outputSufficientSize + ? ErrorStatus::NONE + : ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; + EXPECT_EQ(executionResult.timing, kNoTiming); + return executionStatus; + } + + ErrorStatus executeFenced(const std::shared_ptr& preparedModel, + const Request& request) { + ndk::ScopedFileDescriptor syncFence; + std::shared_ptr fencedCallback; + const auto ret = preparedModel->executeFenced(request, {}, false, kNoDeadline, + kOmittedTimeoutDuration, kNoDuration, + &syncFence, &fencedCallback); + if (!ret.isOk()) { + EXPECT_EQ(ret.getExceptionCode(), EX_SERVICE_SPECIFIC); + return static_cast(ret.getServiceSpecificError()); + } + if (syncFence.get() != -1) { + waitForSyncFence(syncFence.get()); + } + EXPECT_NE(fencedCallback, nullptr); + + ErrorStatus executionStatus = ErrorStatus::GENERAL_FAILURE; + Timing time = kNoTiming; + Timing timeFenced = kNoTiming; + const auto retExecutionInfo = + fencedCallback->getExecutionInfo(&time, &timeFenced, &executionStatus); + EXPECT_TRUE(retExecutionInfo.isOk()); + EXPECT_EQ(time, kNoTiming); + return executionStatus; + } + + const Executor kExecutor = std::get(GetParam()); +}; + +TEST_P(MemoryDomainExecutionTest, InvalidToken) { + auto preparedModel = createConvPreparedModel(kTestOperand); + if (preparedModel == nullptr) return; + + RequestMemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool badDeviceMemory1 = createDeviceMemoryPool(0); // Invalid token. + RequestMemoryPool badDeviceMemory2 = createDeviceMemoryPool(100); // Unknown token. + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, badDeviceMemory1)}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, badDeviceMemory2)}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, badDeviceMemory1)}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, badDeviceMemory2)}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidPreparedModel) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + auto badPreparedModel = createConvPreparedModel(kTestOperand); + if (badPreparedModel == nullptr) return; + + RequestMemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the buffer is not allocated for badPreparedModel. + initializeDeviceMemory(buffer); + testExecution(badPreparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + testExecution(badPreparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidIOIndex) { + auto preparedModel = createConvPreparedModel(kTestOperand, 2); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {}); + if (buffer == nullptr) return; + + RequestMemoryPool sharedMemory1 = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool sharedMemory2 = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool sharedMemory3 = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg1 = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg2 = { + .location = {.poolIndex = 1, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg3 = { + .location = {.poolIndex = 2, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 3}}; + + // This should fail, because the device memory is not allocated for input 1. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, deviceMemoryArg}, + .outputs = {sharedMemoryArg2, sharedMemoryArg3}, + .pools = createRequestMemoryPools(sharedMemory1, sharedMemory2, sharedMemory3, + deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the device memory is not allocated for output 1. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, sharedMemoryArg2}, + .outputs = {sharedMemoryArg3, deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory1, sharedMemory2, sharedMemory3, + deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, InvalidIOType) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [inputBuffer, inputToken] = allocateBuffer(preparedModel, {0}, {}); + auto [outputBuffer, outputToken] = allocateBuffer(preparedModel, {}, {0}); + if (inputBuffer == nullptr || outputBuffer == nullptr) return; + + RequestMemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(inputToken); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the device memory is allocated for input but used as output. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the device memory is allocated for output but used as input. + deviceMemory.set(outputToken); + initializeDeviceMemory(outputBuffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); +} + +TEST_P(MemoryDomainExecutionTest, UninitializedMemory) { + auto preparedModel = createConvPreparedModel(kTestOperand); + auto [buffer, token] = allocateBuffer(preparedModel, {0}, {0}); + if (buffer == nullptr) return; + + RequestMemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + + // This should fail, because the device memory is not initialized. + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::GENERAL_FAILURE); + + // This should initialize the device memory. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::NONE); + + // Test again with initialized device memory. + testExecution(preparedModel, + {.inputs = {deviceMemoryArg}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::NONE); +} + +TEST_P(MemoryDomainExecutionTest, SameRequestMultipleRoles) { + auto preparedModel = createConvPreparedModel(kTestOperand, 2); + auto [buffer, token] = allocateBuffer(preparedModel, {0, 1}, {0, 1}); + if (buffer == nullptr) return; + + RequestMemoryPool sharedMemory1 = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool sharedMemory2 = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(token); + RequestArgument sharedMemoryArg1 = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument sharedMemoryArg2 = { + .location = {.poolIndex = 1, .offset = 0, .length = kTestOperandDataSize}}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 2}}; + + // This should fail, because the same device memory cannot be used for both input and output. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg, sharedMemoryArg1}, + .outputs = {deviceMemoryArg, sharedMemoryArg2}, + .pools = createRequestMemoryPools(sharedMemory1, sharedMemory2, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + // This should fail, because the same device memory cannot be used for multiple outputs. + testExecution(preparedModel, + {.inputs = {sharedMemoryArg1, sharedMemoryArg2}, + .outputs = {deviceMemoryArg, deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory1, sharedMemory2, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + // The same device memory can be used for multiple inputs. + initializeDeviceMemory(buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArg, deviceMemoryArg}, + .outputs = {sharedMemoryArg1, sharedMemoryArg2}, + .pools = createRequestMemoryPools(sharedMemory1, sharedMemory2, deviceMemory)}, + ErrorStatus::NONE); +} + +TEST_P(MemoryDomainExecutionTest, InvalidDimensions) { + // FENCED execution does not support dynamic shape. + if (kExecutor == Executor::FENCED) return; + + TestOperand testOperand = kTestOperand; + testOperand.dimensions[0] = 0; + auto preparedModel = createConvPreparedModel(testOperand); + auto deviceBuffer = allocateBuffer(preparedModel, {0}, {0}, + utils::toSigned(kTestOperand.dimensions).value()); + if (deviceBuffer.buffer == nullptr) return; + + RequestMemoryPool sharedMemory = createSharedMemoryPool(kTestOperandDataSize); + RequestMemoryPool deviceMemory = createDeviceMemoryPool(deviceBuffer.token); + auto badDimensions = utils::toSigned(kTestOperand.dimensions).value(); + badDimensions[0] = 2; + RequestArgument sharedMemoryArg = { + .location = {.poolIndex = 0, .offset = 0, .length = kTestOperandDataSize}, + .dimensions = badDimensions}; + RequestArgument deviceMemoryArg = {.location = {.poolIndex = 1}}; + RequestArgument deviceMemoryArgWithBadDimensions = {.location = {.poolIndex = 1}, + .dimensions = badDimensions}; + + initializeDeviceMemory(deviceBuffer.buffer); + testExecution(preparedModel, + {.inputs = {deviceMemoryArgWithBadDimensions}, + .outputs = {sharedMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArgWithBadDimensions}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::INVALID_ARGUMENT); + + testExecution(preparedModel, + {.inputs = {sharedMemoryArg}, + .outputs = {deviceMemoryArg}, + .pools = createRequestMemoryPools(sharedMemory, deviceMemory)}, + ErrorStatus::GENERAL_FAILURE); +} + +const auto kExecutorChoices = testing::Values(Executor::SYNC, Executor::FENCED); + +std::string printMemoryDomainExecutionTest( + const testing::TestParamInfo& info) { + const auto& [namedDevice, operandType, executor] = info.param; + const std::string type = toString(static_cast(operandType)); + const std::string executorStr = toString(executor); + return gtestCompliantName(getName(namedDevice) + "_" + type + "_" + executorStr); +} + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(MemoryDomainExecutionTest); +INSTANTIATE_TEST_SUITE_P(TestMemoryDomain, MemoryDomainExecutionTest, + testing::Combine(testing::ValuesIn(getNamedDevices()), + kTestOperandTypeChoices, kExecutorChoices), + printMemoryDomainExecutionTest); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/QualityOfServiceTests.cpp b/neuralnetworks/aidl/vts/functional/QualityOfServiceTests.cpp new file mode 100644 index 0000000000..58db98f374 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/QualityOfServiceTests.cpp @@ -0,0 +1,270 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "Utils.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using implementation::PreparedModelCallback; +using test_helper::TestBuffer; +using test_helper::TestModel; + +enum class DeadlineBoundType { NOW, UNLIMITED, SHORT }; +constexpr std::array deadlineBounds = { + DeadlineBoundType::NOW, DeadlineBoundType::UNLIMITED, DeadlineBoundType::SHORT}; +std::string toString(DeadlineBoundType type) { + switch (type) { + case DeadlineBoundType::NOW: + return "NOW"; + case DeadlineBoundType::UNLIMITED: + return "UNLIMITED"; + case DeadlineBoundType::SHORT: + return "SHORT"; + } + LOG(FATAL) << "Unrecognized DeadlineBoundType: " << static_cast(type); + return {}; +} + +constexpr auto kShortDuration = std::chrono::milliseconds{5}; + +using Results = std::tuple, Timing>; +using MaybeResults = std::optional; + +static int64_t makeDeadline(DeadlineBoundType deadlineBoundType) { + const auto getNanosecondsSinceEpoch = [](const auto& time) -> int64_t { + const auto timeSinceEpoch = time.time_since_epoch(); + return std::chrono::duration_cast(timeSinceEpoch).count(); + }; + + std::chrono::steady_clock::time_point timePoint; + switch (deadlineBoundType) { + case DeadlineBoundType::NOW: + timePoint = std::chrono::steady_clock::now(); + break; + case DeadlineBoundType::UNLIMITED: + timePoint = std::chrono::steady_clock::time_point::max(); + break; + case DeadlineBoundType::SHORT: + timePoint = std::chrono::steady_clock::now() + kShortDuration; + break; + } + + return getNanosecondsSinceEpoch(timePoint); +} + +void runPrepareModelTest(const std::shared_ptr& device, const Model& model, + Priority priority, std::optional deadlineBound) { + int64_t deadline = kNoDeadline; + if (deadlineBound.has_value()) { + deadline = makeDeadline(deadlineBound.value()); + } + + // see if service can handle model + std::vector supportedOps; + const auto supportedCallStatus = device->getSupportedOperations(model, &supportedOps); + ASSERT_TRUE(supportedCallStatus.isOk()); + ASSERT_NE(0ul, supportedOps.size()); + const bool fullySupportsModel = + std::all_of(supportedOps.begin(), supportedOps.end(), [](bool valid) { return valid; }); + + // launch prepare model + const std::shared_ptr preparedModelCallback = + ndk::SharedRefBase::make(); + const auto prepareLaunchStatus = + device->prepareModel(model, ExecutionPreference::FAST_SINGLE_ANSWER, priority, deadline, + {}, {}, kEmptyCacheToken, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()) + << "prepareLaunchStatus: " << prepareLaunchStatus.getDescription(); + + // retrieve prepared model + preparedModelCallback->wait(); + const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + const std::shared_ptr preparedModel = preparedModelCallback->getPreparedModel(); + + // The getSupportedOperations call returns a list of operations that are guaranteed not to fail + // if prepareModel is called, and 'fullySupportsModel' is true i.f.f. the entire model is + // guaranteed. If a driver has any doubt that it can prepare an operation, it must return false. + // So here, if a driver isn't sure if it can support an operation, but reports that it + // successfully prepared the model, the test can continue. + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { + ASSERT_EQ(nullptr, preparedModel.get()); + return; + } + + // verify return status + if (!deadlineBound.has_value()) { + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); + } else { + switch (deadlineBound.value()) { + case DeadlineBoundType::NOW: + case DeadlineBoundType::SHORT: + // Either the driver successfully completed the task or it + // aborted and returned MISSED_DEADLINE_*. + EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE || + prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT || + prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT); + break; + case DeadlineBoundType::UNLIMITED: + // If an unlimited deadline is supplied, we expect the execution to + // proceed normally. In this case, check it normally by breaking out + // of the switch statement. + EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); + break; + } + } + ASSERT_EQ(prepareReturnStatus == ErrorStatus::NONE, preparedModel.get() != nullptr); +} + +void runPrepareModelTests(const std::shared_ptr& device, const Model& model) { + // test priority + for (auto priority : ndk::enum_range{}) { + SCOPED_TRACE("priority: " + toString(priority)); + if (priority == kDefaultPriority) continue; + runPrepareModelTest(device, model, priority, {}); + } + + // test deadline + for (auto deadlineBound : deadlineBounds) { + SCOPED_TRACE("deadlineBound: " + toString(deadlineBound)); + runPrepareModelTest(device, model, kDefaultPriority, deadlineBound); + } +} + +static MaybeResults executeSynchronously(const std::shared_ptr& preparedModel, + const Request& request, int64_t deadline) { + SCOPED_TRACE("synchronous"); + const bool measure = false; + + // run execution + ExecutionResult executionResult; + const auto ret = preparedModel->executeSynchronously(request, measure, deadline, + kOmittedTimeoutDuration, &executionResult); + EXPECT_TRUE(ret.isOk() || ret.getExceptionCode() == EX_SERVICE_SPECIFIC) + << ret.getDescription(); + if (!ret.isOk()) { + if (ret.getExceptionCode() != EX_SERVICE_SPECIFIC) { + return std::nullopt; + } + return MaybeResults( + {static_cast(ret.getServiceSpecificError()), {}, kNoTiming}); + } + + // return results + return MaybeResults({executionResult.outputSufficientSize + ? ErrorStatus::NONE + : ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, + std::move(executionResult.outputShapes), executionResult.timing}); +} + +void runExecutionTest(const std::shared_ptr& preparedModel, + const TestModel& testModel, const Request& request, + const ExecutionContext& context, DeadlineBoundType deadlineBound) { + const auto deadline = makeDeadline(deadlineBound); + + // Perform execution and unpack results. + const auto results = executeSynchronously(preparedModel, request, deadline); + if (!results.has_value()) return; + const auto& [status, outputShapes, timing] = results.value(); + + // Verify no timing information was returned + EXPECT_EQ(timing, kNoTiming); + + // Validate deadline information if applicable. + switch (deadlineBound) { + case DeadlineBoundType::NOW: + case DeadlineBoundType::SHORT: + // Either the driver successfully completed the task or it + // aborted and returned MISSED_DEADLINE_*. + ASSERT_TRUE(status == ErrorStatus::NONE || + status == ErrorStatus::MISSED_DEADLINE_TRANSIENT || + status == ErrorStatus::MISSED_DEADLINE_PERSISTENT); + break; + case DeadlineBoundType::UNLIMITED: + // If an unlimited deadline is supplied, we expect the execution to + // proceed normally. In this case, check it normally by breaking out + // of the switch statement. + ASSERT_EQ(ErrorStatus::NONE, status); + break; + } + + // If the model output operands are fully specified, outputShapes must be either + // either empty, or have the same number of elements as the number of outputs. + ASSERT_TRUE(outputShapes.size() == 0 || + outputShapes.size() == testModel.main.outputIndexes.size()); + + // Go through all outputs, check returned output shapes. + for (uint32_t i = 0; i < outputShapes.size(); i++) { + EXPECT_TRUE(outputShapes[i].isSufficient); + const auto expect = + utils::toSigned(testModel.main.operands[testModel.main.outputIndexes[i]].dimensions) + .value(); + const std::vector& actual = outputShapes[i].dimensions; + EXPECT_EQ(expect, actual); + } + + // Retrieve execution results. + const std::vector outputs = context.getOutputBuffers(request); + + // We want "close-enough" results. + if (status == ErrorStatus::NONE) { + checkResults(testModel, outputs); + } +} + +void runExecutionTests(const std::shared_ptr& preparedModel, + const TestModel& testModel, const Request& request, + const ExecutionContext& context) { + for (auto deadlineBound : deadlineBounds) { + runExecutionTest(preparedModel, testModel, request, context, deadlineBound); + } +} + +void runTests(const std::shared_ptr& device, const TestModel& testModel) { + // setup + const Model model = createModel(testModel); + + // run prepare model tests + runPrepareModelTests(device, model); + + // prepare model + std::shared_ptr preparedModel; + createPreparedModel(device, model, &preparedModel); + if (preparedModel == nullptr) return; + + // run execution tests + ExecutionContext context; + const Request request = context.createRequest(testModel); + runExecutionTests(preparedModel, testModel, request, context); +} + +class DeadlineTest : public GeneratedTestBase {}; + +TEST_P(DeadlineTest, Test) { + runTests(kDevice, kTestModel); +} + +INSTANTIATE_GENERATED_TEST(DeadlineTest, + [](const TestModel& testModel) { return !testModel.expectFailure; }); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/TestAssertions.cpp b/neuralnetworks/aidl/vts/functional/TestAssertions.cpp new file mode 100644 index 0000000000..a9e945608c --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/TestAssertions.cpp @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include + +namespace aidl::android::hardware::neuralnetworks { + +namespace nn = ::android::nn; + +static_assert(static_cast(IPreparedModel::DEFAULT_LOOP_TIMEOUT_DURATION_NS) == + nn::operation_while::kTimeoutNsDefault); +static_assert(static_cast(IPreparedModel::MAXIMUM_LOOP_TIMEOUT_DURATION_NS) == + nn::operation_while::kTimeoutNsMaximum); + +// Make sure that the HIDL enums are compatible with the values defined in +// frameworks/ml/nn/tools/test_generator/test_harness/include/TestHarness.h. +using namespace test_helper; +#define CHECK_TEST_ENUM(EnumType, enumValue) \ + static_assert(static_cast(Test##EnumType::enumValue) == EnumType::enumValue) + +CHECK_TEST_ENUM(OperandType, FLOAT32); +CHECK_TEST_ENUM(OperandType, INT32); +CHECK_TEST_ENUM(OperandType, UINT32); +CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT32); +CHECK_TEST_ENUM(OperandType, TENSOR_INT32); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM); +CHECK_TEST_ENUM(OperandType, BOOL); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_SYMM); +CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT16); +CHECK_TEST_ENUM(OperandType, TENSOR_BOOL8); +CHECK_TEST_ENUM(OperandType, FLOAT16); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM_PER_CHANNEL); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_ASYMM); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM); +CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM_SIGNED); + +CHECK_TEST_ENUM(OperationType, ADD); +CHECK_TEST_ENUM(OperationType, AVERAGE_POOL_2D); +CHECK_TEST_ENUM(OperationType, CONCATENATION); +CHECK_TEST_ENUM(OperationType, CONV_2D); +CHECK_TEST_ENUM(OperationType, DEPTHWISE_CONV_2D); +CHECK_TEST_ENUM(OperationType, DEPTH_TO_SPACE); +CHECK_TEST_ENUM(OperationType, DEQUANTIZE); +CHECK_TEST_ENUM(OperationType, EMBEDDING_LOOKUP); +CHECK_TEST_ENUM(OperationType, FLOOR); +CHECK_TEST_ENUM(OperationType, FULLY_CONNECTED); +CHECK_TEST_ENUM(OperationType, HASHTABLE_LOOKUP); +CHECK_TEST_ENUM(OperationType, L2_NORMALIZATION); +CHECK_TEST_ENUM(OperationType, L2_POOL_2D); +CHECK_TEST_ENUM(OperationType, LOCAL_RESPONSE_NORMALIZATION); +CHECK_TEST_ENUM(OperationType, LOGISTIC); +CHECK_TEST_ENUM(OperationType, LSH_PROJECTION); +CHECK_TEST_ENUM(OperationType, LSTM); +CHECK_TEST_ENUM(OperationType, MAX_POOL_2D); +CHECK_TEST_ENUM(OperationType, MUL); +CHECK_TEST_ENUM(OperationType, RELU); +CHECK_TEST_ENUM(OperationType, RELU1); +CHECK_TEST_ENUM(OperationType, RELU6); +CHECK_TEST_ENUM(OperationType, RESHAPE); +CHECK_TEST_ENUM(OperationType, RESIZE_BILINEAR); +CHECK_TEST_ENUM(OperationType, RNN); +CHECK_TEST_ENUM(OperationType, SOFTMAX); +CHECK_TEST_ENUM(OperationType, SPACE_TO_DEPTH); +CHECK_TEST_ENUM(OperationType, SVDF); +CHECK_TEST_ENUM(OperationType, TANH); +CHECK_TEST_ENUM(OperationType, BATCH_TO_SPACE_ND); +CHECK_TEST_ENUM(OperationType, DIV); +CHECK_TEST_ENUM(OperationType, MEAN); +CHECK_TEST_ENUM(OperationType, PAD); +CHECK_TEST_ENUM(OperationType, SPACE_TO_BATCH_ND); +CHECK_TEST_ENUM(OperationType, SQUEEZE); +CHECK_TEST_ENUM(OperationType, STRIDED_SLICE); +CHECK_TEST_ENUM(OperationType, SUB); +CHECK_TEST_ENUM(OperationType, TRANSPOSE); +CHECK_TEST_ENUM(OperationType, ABS); +CHECK_TEST_ENUM(OperationType, ARGMAX); +CHECK_TEST_ENUM(OperationType, ARGMIN); +CHECK_TEST_ENUM(OperationType, AXIS_ALIGNED_BBOX_TRANSFORM); +CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_LSTM); +CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_RNN); +CHECK_TEST_ENUM(OperationType, BOX_WITH_NMS_LIMIT); +CHECK_TEST_ENUM(OperationType, CAST); +CHECK_TEST_ENUM(OperationType, CHANNEL_SHUFFLE); +CHECK_TEST_ENUM(OperationType, DETECTION_POSTPROCESSING); +CHECK_TEST_ENUM(OperationType, EQUAL); +CHECK_TEST_ENUM(OperationType, EXP); +CHECK_TEST_ENUM(OperationType, EXPAND_DIMS); +CHECK_TEST_ENUM(OperationType, GATHER); +CHECK_TEST_ENUM(OperationType, GENERATE_PROPOSALS); +CHECK_TEST_ENUM(OperationType, GREATER); +CHECK_TEST_ENUM(OperationType, GREATER_EQUAL); +CHECK_TEST_ENUM(OperationType, GROUPED_CONV_2D); +CHECK_TEST_ENUM(OperationType, HEATMAP_MAX_KEYPOINT); +CHECK_TEST_ENUM(OperationType, INSTANCE_NORMALIZATION); +CHECK_TEST_ENUM(OperationType, LESS); +CHECK_TEST_ENUM(OperationType, LESS_EQUAL); +CHECK_TEST_ENUM(OperationType, LOG); +CHECK_TEST_ENUM(OperationType, LOGICAL_AND); +CHECK_TEST_ENUM(OperationType, LOGICAL_NOT); +CHECK_TEST_ENUM(OperationType, LOGICAL_OR); +CHECK_TEST_ENUM(OperationType, LOG_SOFTMAX); +CHECK_TEST_ENUM(OperationType, MAXIMUM); +CHECK_TEST_ENUM(OperationType, MINIMUM); +CHECK_TEST_ENUM(OperationType, NEG); +CHECK_TEST_ENUM(OperationType, NOT_EQUAL); +CHECK_TEST_ENUM(OperationType, PAD_V2); +CHECK_TEST_ENUM(OperationType, POW); +CHECK_TEST_ENUM(OperationType, PRELU); +CHECK_TEST_ENUM(OperationType, QUANTIZE); +CHECK_TEST_ENUM(OperationType, QUANTIZED_16BIT_LSTM); +CHECK_TEST_ENUM(OperationType, RANDOM_MULTINOMIAL); +CHECK_TEST_ENUM(OperationType, REDUCE_ALL); +CHECK_TEST_ENUM(OperationType, REDUCE_ANY); +CHECK_TEST_ENUM(OperationType, REDUCE_MAX); +CHECK_TEST_ENUM(OperationType, REDUCE_MIN); +CHECK_TEST_ENUM(OperationType, REDUCE_PROD); +CHECK_TEST_ENUM(OperationType, REDUCE_SUM); +CHECK_TEST_ENUM(OperationType, ROI_ALIGN); +CHECK_TEST_ENUM(OperationType, ROI_POOLING); +CHECK_TEST_ENUM(OperationType, RSQRT); +CHECK_TEST_ENUM(OperationType, SELECT); +CHECK_TEST_ENUM(OperationType, SIN); +CHECK_TEST_ENUM(OperationType, SLICE); +CHECK_TEST_ENUM(OperationType, SPLIT); +CHECK_TEST_ENUM(OperationType, SQRT); +CHECK_TEST_ENUM(OperationType, TILE); +CHECK_TEST_ENUM(OperationType, TOPK_V2); +CHECK_TEST_ENUM(OperationType, TRANSPOSE_CONV_2D); +CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_LSTM); +CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_RNN); +CHECK_TEST_ENUM(OperationType, RESIZE_NEAREST_NEIGHBOR); + +#undef CHECK_TEST_ENUM + +} // namespace aidl::android::hardware::neuralnetworks diff --git a/neuralnetworks/aidl/vts/functional/TestMain.cpp b/neuralnetworks/aidl/vts/functional/TestMain.cpp new file mode 100644 index 0000000000..1d58608fa3 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/TestMain.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "LogTestCaseToLogcat.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + testing::UnitTest::GetInstance()->listeners().Append( + new aidl::android::hardware::neuralnetworks::LogTestCaseToLogcat()); + ABinderProcess_startThreadPool(); + return RUN_ALL_TESTS(); +} diff --git a/neuralnetworks/aidl/vts/functional/Utils.cpp b/neuralnetworks/aidl/vts/functional/Utils.cpp new file mode 100644 index 0000000000..14a496a303 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/Utils.cpp @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Utils.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace aidl::android::hardware::neuralnetworks { + +using test_helper::TestBuffer; +using test_helper::TestModel; + +uint32_t sizeOfData(OperandType type) { + switch (type) { + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::TENSOR_FLOAT32: + case OperandType::TENSOR_INT32: + return 4; + case OperandType::TENSOR_QUANT16_SYMM: + case OperandType::TENSOR_FLOAT16: + case OperandType::FLOAT16: + case OperandType::TENSOR_QUANT16_ASYMM: + return 2; + case OperandType::TENSOR_QUANT8_ASYMM: + case OperandType::BOOL: + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: + case OperandType::TENSOR_QUANT8_SYMM: + case OperandType::TENSOR_QUANT8_ASYMM_SIGNED: + return 1; + case OperandType::SUBGRAPH: + return 0; + default: + CHECK(false) << "Invalid OperandType " << static_cast(type); + return 0; + } +} + +static bool isTensor(OperandType type) { + switch (type) { + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::FLOAT16: + case OperandType::BOOL: + case OperandType::SUBGRAPH: + return false; + case OperandType::TENSOR_FLOAT32: + case OperandType::TENSOR_INT32: + case OperandType::TENSOR_QUANT16_SYMM: + case OperandType::TENSOR_FLOAT16: + case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT8_ASYMM: + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: + case OperandType::TENSOR_QUANT8_SYMM: + case OperandType::TENSOR_QUANT8_ASYMM_SIGNED: + return true; + default: + CHECK(false) << "Invalid OperandType " << static_cast(type); + return false; + } +} + +uint32_t sizeOfData(const Operand& operand) { + const uint32_t dataSize = sizeOfData(operand.type); + if (isTensor(operand.type) && operand.dimensions.size() == 0) return 0; + return std::accumulate(operand.dimensions.begin(), operand.dimensions.end(), dataSize, + std::multiplies<>{}); +} + +std::unique_ptr TestAshmem::create(uint32_t size) { + auto ashmem = std::make_unique(size); + return ashmem->mIsValid ? std::move(ashmem) : nullptr; +} + +void TestAshmem::initialize(uint32_t size) { + mIsValid = false; + ASSERT_GT(size, 0); + const auto sharedMemory = nn::createSharedMemory(size).value(); + mMappedMemory = nn::map(sharedMemory).value(); + mPtr = static_cast(std::get(mMappedMemory.pointer)); + CHECK_NE(mPtr, nullptr); + mAidlMemory = utils::convert(sharedMemory).value(); + mIsValid = true; +} + +std::unique_ptr TestBlobAHWB::create(uint32_t size) { + auto ahwb = std::make_unique(size); + return ahwb->mIsValid ? std::move(ahwb) : nullptr; +} + +void TestBlobAHWB::initialize(uint32_t size) { + mIsValid = false; + ASSERT_GT(size, 0); + const auto usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN; + const AHardwareBuffer_Desc desc = { + .width = size, + .height = 1, + .layers = 1, + .format = AHARDWAREBUFFER_FORMAT_BLOB, + .usage = usage, + .stride = size, + }; + + ASSERT_EQ(AHardwareBuffer_allocate(&desc, &mAhwb), 0); + ASSERT_NE(mAhwb, nullptr); + + const auto sharedMemory = nn::createSharedMemoryFromAHWB(*mAhwb).value(); + mMapping = nn::map(sharedMemory).value(); + mPtr = static_cast(std::get(mMapping.pointer)); + CHECK_NE(mPtr, nullptr); + mAidlMemory = utils::convert(sharedMemory).value(); + + mIsValid = true; +} + +TestBlobAHWB::~TestBlobAHWB() { + if (mAhwb) { + AHardwareBuffer_unlock(mAhwb, nullptr); + AHardwareBuffer_release(mAhwb); + } +} + +std::string gtestCompliantName(std::string name) { + // gtest test names must only contain alphanumeric characters + std::replace_if( + name.begin(), name.end(), [](char c) { return !std::isalnum(c); }, '_'); + return name; +} + +::std::ostream& operator<<(::std::ostream& os, ErrorStatus errorStatus) { + return os << toString(errorStatus); +} + +Request ExecutionContext::createRequest(const TestModel& testModel, MemoryType memoryType) { + CHECK(memoryType == MemoryType::ASHMEM || memoryType == MemoryType::BLOB_AHWB); + + // Model inputs. + std::vector inputs(testModel.main.inputIndexes.size()); + size_t inputSize = 0; + for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { + const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; + if (op.data.size() == 0) { + // Omitted input. + inputs[i] = {.hasNoValue = true}; + } else { + DataLocation loc = {.poolIndex = kInputPoolIndex, + .offset = static_cast(inputSize), + .length = static_cast(op.data.size())}; + inputSize += op.data.alignedSize(); + inputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + } + } + + // Model outputs. + std::vector outputs(testModel.main.outputIndexes.size()); + size_t outputSize = 0; + for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) { + const auto& op = testModel.main.operands[testModel.main.outputIndexes[i]]; + + // In the case of zero-sized output, we should at least provide a one-byte buffer. + // This is because zero-sized tensors are only supported internally to the driver, or + // reported in output shapes. It is illegal for the client to pre-specify a zero-sized + // tensor as model output. Otherwise, we will have two semantic conflicts: + // - "Zero dimension" conflicts with "unspecified dimension". + // - "Omitted operand buffer" conflicts with "zero-sized operand buffer". + size_t bufferSize = std::max(op.data.size(), 1); + + DataLocation loc = {.poolIndex = kOutputPoolIndex, + .offset = static_cast(outputSize), + .length = static_cast(bufferSize)}; + outputSize += op.data.size() == 0 ? TestBuffer::kAlignment : op.data.alignedSize(); + outputs[i] = {.hasNoValue = false, .location = loc, .dimensions = {}}; + } + + // Allocate memory pools. + if (memoryType == MemoryType::ASHMEM) { + mInputMemory = TestAshmem::create(inputSize); + mOutputMemory = TestAshmem::create(outputSize); + } else { + mInputMemory = TestBlobAHWB::create(inputSize); + mOutputMemory = TestBlobAHWB::create(outputSize); + } + CHECK_NE(mInputMemory, nullptr); + CHECK_NE(mOutputMemory, nullptr); + + auto copiedInputMemory = utils::clone(*mInputMemory->getAidlMemory()); + CHECK(copiedInputMemory.has_value()) << copiedInputMemory.error().message; + auto copiedOutputMemory = utils::clone(*mOutputMemory->getAidlMemory()); + CHECK(copiedOutputMemory.has_value()) << copiedOutputMemory.error().message; + + std::vector pools; + pools.push_back(RequestMemoryPool::make( + std::move(copiedInputMemory).value())); + pools.push_back(RequestMemoryPool::make( + std::move(copiedOutputMemory).value())); + + // Copy input data to the memory pool. + uint8_t* inputPtr = mInputMemory->getPointer(); + for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) { + const auto& op = testModel.main.operands[testModel.main.inputIndexes[i]]; + if (op.data.size() > 0) { + const uint8_t* begin = op.data.get(); + const uint8_t* end = begin + op.data.size(); + std::copy(begin, end, inputPtr + inputs[i].location.offset); + } + } + + return {.inputs = std::move(inputs), .outputs = std::move(outputs), .pools = std::move(pools)}; +} + +std::vector ExecutionContext::getOutputBuffers(const Request& request) const { + // Copy out output results. + uint8_t* outputPtr = mOutputMemory->getPointer(); + std::vector outputBuffers; + for (const auto& output : request.outputs) { + outputBuffers.emplace_back(output.location.length, outputPtr + output.location.offset); + } + return outputBuffers; +} + +} // namespace aidl::android::hardware::neuralnetworks diff --git a/neuralnetworks/aidl/vts/functional/Utils.h b/neuralnetworks/aidl/vts/functional/Utils.h new file mode 100644 index 0000000000..266301ca97 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/Utils.h @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_NEURALNETWORKS_AIDL_UTILS_H +#define ANDROID_HARDWARE_NEURALNETWORKS_AIDL_UTILS_H + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace aidl::android::hardware::neuralnetworks { + +namespace nn = ::android::nn; + +inline constexpr Priority kDefaultPriority = Priority::MEDIUM; + +inline constexpr Timing kNoTiming = {.timeOnDevice = -1, .timeInDriver = -1}; +inline constexpr int64_t kNoDeadline = -1; +inline constexpr int64_t kOmittedTimeoutDuration = -1; +inline constexpr int64_t kNoDuration = -1; +inline const std::vector kEmptyCacheToken(IDevice::BYTE_SIZE_OF_CACHE_TOKEN); + +// Returns the amount of space needed to store a value of the specified type. +// +// Aborts if the specified type is an extension type or OEM type. +uint32_t sizeOfData(OperandType type); + +// Returns the amount of space needed to store a value of the dimensions and +// type of this operand. For a non-extension, non-OEM tensor with unspecified +// rank or at least one unspecified dimension, returns zero. +// +// Aborts if the specified type is an extension type or OEM type. +uint32_t sizeOfData(const Operand& operand); + +// Convenience class to manage the lifetime of memory resources. +class TestMemoryBase { + DISALLOW_COPY_AND_ASSIGN(TestMemoryBase); + + public: + TestMemoryBase() = default; + virtual ~TestMemoryBase() = default; + uint8_t* getPointer() const { return mPtr; } + const Memory* getAidlMemory() const { return &mAidlMemory; } + + protected: + uint8_t* mPtr = nullptr; + Memory mAidlMemory; + bool mIsValid = false; +}; + +class TestAshmem : public TestMemoryBase { + public: + static std::unique_ptr create(uint32_t size); + + // Prefer TestAshmem::create. + // The constructor calls initialize, which constructs the memory resources. This is a workaround + // that gtest macros cannot be used directly in a constructor. + TestAshmem(uint32_t size) { initialize(size); } + + private: + void initialize(uint32_t size); + nn::Mapping mMappedMemory; +}; + +class TestBlobAHWB : public TestMemoryBase { + public: + static std::unique_ptr create(uint32_t size); + + // Prefer TestBlobAHWB::create. + // The constructor calls initialize, which constructs the memory resources. This is a + // workaround that gtest macros cannot be used directly in a constructor. + TestBlobAHWB(uint32_t size) { initialize(size); } + ~TestBlobAHWB(); + + private: + void initialize(uint32_t size); + AHardwareBuffer* mAhwb = nullptr; + nn::Mapping mMapping; +}; + +enum class MemoryType { ASHMEM, BLOB_AHWB, DEVICE }; + +// Manages the lifetime of memory resources used in an execution. +class ExecutionContext { + DISALLOW_COPY_AND_ASSIGN(ExecutionContext); + + public: + static constexpr uint32_t kInputPoolIndex = 0; + static constexpr uint32_t kOutputPoolIndex = 1; + + ExecutionContext() = default; + + // Create HIDL Request from the TestModel struct. + Request createRequest(const test_helper::TestModel& testModel, + MemoryType memoryType = MemoryType::ASHMEM); + + // After execution, copy out output results from the output memory pool. + std::vector getOutputBuffers(const Request& request) const; + + private: + std::unique_ptr mInputMemory, mOutputMemory; +}; + +template +using Named = std::pair; + +template +const std::string& getName(const Named& namedData) { + return namedData.first; +} + +template +const Type& getData(const Named& namedData) { + return namedData.second; +} + +std::string gtestCompliantName(std::string name); + +// pretty-print values for error messages +::std::ostream& operator<<(::std::ostream& os, ErrorStatus errorStatus); + +} // namespace aidl::android::hardware::neuralnetworks + +#endif // ANDROID_HARDWARE_NEURALNETWORKS_AIDL_UTILS_H diff --git a/neuralnetworks/aidl/vts/functional/ValidateModel.cpp b/neuralnetworks/aidl/vts/functional/ValidateModel.cpp new file mode 100644 index 0000000000..b84d981abd --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/ValidateModel.cpp @@ -0,0 +1,1338 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using common::NativeHandle; +using implementation::PreparedModelCallback; + +using PrepareModelMutation = std::function; + +///////////////////////// UTILITY FUNCTIONS ///////////////////////// + +static void validateGetSupportedOperations(const std::shared_ptr& device, + const std::string& message, const Model& model) { + SCOPED_TRACE(message + " [getSupportedOperations]"); + + std::vector supported; + const auto retStatus = device->getSupportedOperations(model, &supported); + + ASSERT_FALSE(retStatus.isOk()); + ASSERT_EQ(retStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(retStatus.getServiceSpecificError()), + ErrorStatus::INVALID_ARGUMENT); +} + +static void validatePrepareModel(const std::shared_ptr& device, const std::string& message, + const Model& model, ExecutionPreference preference, + Priority priority) { + SCOPED_TRACE(message + " [prepareModel]"); + + std::shared_ptr preparedModelCallback = + ndk::SharedRefBase::make(); + const auto prepareLaunchStatus = + device->prepareModel(model, preference, priority, kNoDeadline, {}, {}, kEmptyCacheToken, + preparedModelCallback); + ASSERT_FALSE(prepareLaunchStatus.isOk()); + ASSERT_EQ(prepareLaunchStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(prepareLaunchStatus.getServiceSpecificError()), + ErrorStatus::INVALID_ARGUMENT); + + preparedModelCallback->wait(); + ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus); + std::shared_ptr preparedModel = preparedModelCallback->getPreparedModel(); + ASSERT_EQ(nullptr, preparedModel.get()); +} + +static bool validExecutionPreference(ExecutionPreference preference) { + return preference == ExecutionPreference::LOW_POWER || + preference == ExecutionPreference::FAST_SINGLE_ANSWER || + preference == ExecutionPreference::SUSTAINED_SPEED; +} + +static bool validExecutionPriority(Priority priority) { + return priority == Priority::LOW || priority == Priority::MEDIUM || priority == Priority::HIGH; +} + +// Primary validation function. This function will take a valid model, apply a +// mutation to invalidate the model, the execution preference, or the priority, +// then pass these to supportedOperations and/or prepareModel if that method is +// called with an invalid argument. +static void validate(const std::shared_ptr& device, const std::string& message, + const Model& originalModel, const PrepareModelMutation& mutate) { + Model model = utils::clone(originalModel).value(); + ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER; + Priority priority = kDefaultPriority; + mutate(&model, &preference, &priority); + + if (validExecutionPreference(preference) && validExecutionPriority(priority)) { + validateGetSupportedOperations(device, message, model); + } + + validatePrepareModel(device, message, model, preference, priority); +} + +static uint32_t addOperand(Model* model) { + model->main.operands.push_back({ + .type = OperandType::INT32, + .dimensions = {}, + .scale = 0.0f, + .zeroPoint = 0, + .lifetime = OperandLifeTime::SUBGRAPH_INPUT, + .location = {.poolIndex = 0, .offset = 0, .length = 0}, + }); + return model->main.operands.size() - 1; +} + +static uint32_t addOperand(Model* model, OperandLifeTime lifetime) { + uint32_t index = addOperand(model); + model->main.operands[index].lifetime = lifetime; + return index; +} + +// If we introduce a CONSTANT_COPY for an operand of size operandSize, +// how much will this increase the size of the model? This assumes +// that we can (re)use all of model.operandValues for the operand +// value. +static size_t constantCopyExtraSize(const Model& model, size_t operandSize) { + const size_t operandValuesSize = model.operandValues.size(); + return (operandValuesSize < operandSize) ? (operandSize - operandValuesSize) : 0; +} + +// Highly specialized utility routine for converting an operand to +// CONSTANT_COPY lifetime. +// +// Expects that: +// - operand has a known size +// - operand->lifetime has already been set to CONSTANT_COPY +// - operand->location has been zeroed out +// +// Does the following: +// - initializes operand->location to point to the beginning of model->operandValues +// - resizes model->operandValues (if necessary) to be large enough for the operand +// value, padding it with zeroes on the end +// +// Potential problem: +// By changing the operand to CONSTANT_COPY lifetime, this function is effectively initializing the +// operand with unspecified (but deterministic) data. This means that the model may be invalidated +// in two ways: not only is the lifetime of CONSTANT_COPY invalid, but the operand's value in the +// graph may also be invalid (e.g., if the operand is used as an activation code and has an invalid +// value). For now, this should be fine because it just means we're not testing what we think we're +// testing in certain cases; but we can handwave this and assume we're probabilistically likely to +// exercise the validation code over the span of the entire test set and operand space. +// +// Aborts if the specified operand type is an extension type or OEM type. +static void becomeConstantCopy(Model* model, Operand* operand) { + // sizeOfData will abort if the specified type is an extension type or OEM type. + const size_t sizeOfOperand = sizeOfData(*operand); + EXPECT_NE(sizeOfOperand, size_t(0)); + operand->location.poolIndex = 0; + operand->location.offset = 0; + operand->location.length = sizeOfOperand; + if (model->operandValues.size() < sizeOfOperand) { + model->operandValues.resize(sizeOfOperand); + } +} + +// The sizeForBinder() functions estimate the size of the +// representation of a value when sent to binder. It's probably a bit +// of an under-estimate, because we don't know the size of the +// metadata in the binder format (e.g., representation of the size of +// a vector); but at least it adds up "big" things like vector +// contents. However, it doesn't treat inter-field or end-of-struct +// padding in a methodical way -- there's no attempt to be consistent +// in whether or not padding in the native (C++) representation +// contributes to the estimated size for the binder representation; +// and there's no attempt to understand what padding (if any) is +// needed in the binder representation. +// +// This assumes that non-metadata uses a fixed length encoding (e.g., +// a uint32_t is always encoded in sizeof(uint32_t) bytes, rather than +// using an encoding whose length is related to the magnitude of the +// encoded value). + +template +static size_t sizeForBinder(const Type& val) { + static_assert(std::is_trivially_copyable_v>, + "expected a trivially copyable type"); + return sizeof(val); +} + +template +static size_t sizeForBinder(const std::vector& vec) { + return std::accumulate(vec.begin(), vec.end(), 0, + [](size_t acc, const Type& x) { return acc + sizeForBinder(x); }); +} + +template <> +size_t sizeForBinder(const SymmPerChannelQuantParams& symmPerChannelQuantParams) { + size_t size = 0; + + size += sizeForBinder(symmPerChannelQuantParams.scales); + size += sizeForBinder(symmPerChannelQuantParams.channelDim); + + return size; +} + +template <> +size_t sizeForBinder(const std::optional& optionalExtraParams) { + if (!optionalExtraParams.has_value()) { + return 0; + } + const auto& extraParams = optionalExtraParams.value(); + using Tag = OperandExtraParams::Tag; + switch (extraParams.getTag()) { + case Tag::channelQuant: + return sizeForBinder(extraParams.get()); + case Tag::extension: + return sizeForBinder(extraParams.get()); + } + LOG(FATAL) << "Unrecognized extraParams tag: " << static_cast(extraParams.getTag()); + return 0; +} + +template <> +size_t sizeForBinder(const Operand& operand) { + size_t size = 0; + + size += sizeForBinder(operand.type); + size += sizeForBinder(operand.dimensions); + size += sizeForBinder(operand.scale); + size += sizeForBinder(operand.zeroPoint); + size += sizeForBinder(operand.lifetime); + size += sizeForBinder(operand.location); + size += sizeForBinder(operand.extraParams); + + return size; +} + +template <> +size_t sizeForBinder(const Operation& operation) { + size_t size = 0; + + size += sizeForBinder(operation.type); + size += sizeForBinder(operation.inputs); + size += sizeForBinder(operation.outputs); + + return size; +} + +template <> +size_t sizeForBinder(const std::string& name) { + return name.size(); +} + +template <> +size_t sizeForBinder(const Memory& memory) { + // This is just a guess. + + size_t size = 0; + const NativeHandle& handle = memory.handle; + size += sizeof(decltype(handle.fds)::value_type) * handle.fds.size(); + size += sizeof(decltype(handle.ints)::value_type) * handle.ints.size(); + size += sizeForBinder(memory.name); + size += sizeof(memory); + + return size; +} + +template <> +size_t sizeForBinder(const Subgraph& subgraph) { + size_t size = 0; + + size += sizeForBinder(subgraph.operands); + size += sizeForBinder(subgraph.operations); + size += sizeForBinder(subgraph.inputIndexes); + size += sizeForBinder(subgraph.outputIndexes); + + return size; +} + +template <> +size_t sizeForBinder(const ExtensionNameAndPrefix& extensionNameToPrefix) { + size_t size = 0; + + size += sizeForBinder(extensionNameToPrefix.name); + size += sizeForBinder(extensionNameToPrefix.prefix); + + return size; +} + +template <> +size_t sizeForBinder(const Model& model) { + size_t size = 0; + + size += sizeForBinder(model.main); + size += sizeForBinder(model.referenced); + size += sizeForBinder(model.operandValues); + size += sizeForBinder(model.pools); + size += sizeForBinder(model.relaxComputationFloat32toFloat16); + size += sizeForBinder(model.extensionNameToPrefix); + + return size; +} + +// https://developer.android.com/reference/android/os/TransactionTooLargeException.html +// +// "The Binder transaction buffer has a limited fixed size, +// currently 1Mb, which is shared by all transactions in progress +// for the process." +// +// Will our representation fit under this limit? There are two complications: +// - Our representation size is just approximate (see sizeForBinder()). +// - This object may not be the only occupant of the Binder transaction buffer. +// So we'll be very conservative: We want the representation size to be no +// larger than half the transaction buffer size. +// +// If our representation grows large enough that it still fits within +// the transaction buffer but combined with other transactions may +// exceed the buffer size, then we may see intermittent HAL transport +// errors. +static bool exceedsBinderSizeLimit(size_t representationSize) { + // Instead of using this fixed buffer size, we might instead be able to use + // ProcessState::self()->getMmapSize(). However, this has a potential + // problem: The binder/mmap size of the current process does not necessarily + // indicate the binder/mmap size of the service (i.e., the other process). + // The only way it would be a good indication is if both the current process + // and the service use the default size. + static const size_t kHalfBufferSize = 1024 * 1024 / 2; + + return representationSize > kHalfBufferSize; +} + +///////////////////////// VALIDATE EXECUTION ORDER //////////////////////////// + +static void mutateExecutionOrderTest(const std::shared_ptr& device, const Model& model, + const std::vector& numberOfConsumers) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + const Operation& operationObj = model.main.operations[operation]; + for (uint32_t input : operationObj.inputs) { + if (model.main.operands[input].lifetime == OperandLifeTime::TEMPORARY_VARIABLE || + model.main.operands[input].lifetime == OperandLifeTime::SUBGRAPH_OUTPUT) { + // This operation reads an operand written by some + // other operation. Move this operation to the + // beginning of the sequence, ensuring that it reads + // the operand before that operand is written, thereby + // violating execution order rules. + const std::string message = "mutateExecutionOrderTest: operation " + + std::to_string(operation) + " is a reader"; + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + auto& operations = model->main.operations; + std::rotate(operations.begin(), operations.begin() + operation, + operations.begin() + operation + 1); + }); + break; // only need to do this once per operation + } + } + for (uint32_t output : operationObj.outputs) { + if (numberOfConsumers[output] > 0) { + // This operation writes an operand read by some other + // operation. Move this operation to the end of the + // sequence, ensuring that it writes the operand after + // that operand is read, thereby violating execution + // order rules. + const std::string message = "mutateExecutionOrderTest: operation " + + std::to_string(operation) + " is a writer"; + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + auto& operations = model->main.operations; + std::rotate(operations.begin() + operation, + operations.begin() + operation + 1, operations.end()); + }); + break; // only need to do this once per operation + } + } + } +} + +///////////////////////// VALIDATE MODEL OPERAND TYPE ///////////////////////// + +static const int32_t invalidOperandTypes[] = { + -1, + static_cast(*(ndk::enum_range().end() - 1)) + 1, +}; + +static void mutateOperandTypeTest(const std::shared_ptr& device, const Model& model) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + for (int32_t invalidOperandType : invalidOperandTypes) { + const std::string message = "mutateOperandTypeTest: operand " + + std::to_string(operand) + " set to value " + + std::to_string(invalidOperandType); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].type = + static_cast(invalidOperandType); + }); + } + } +} + +///////////////////////// VALIDATE OPERAND RANK ///////////////////////// + +static uint32_t getInvalidRank(OperandType type) { + switch (type) { + case OperandType::FLOAT16: + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::BOOL: + return 1; + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_FLOAT16: + case OperandType::TENSOR_FLOAT32: + case OperandType::TENSOR_INT32: + case OperandType::TENSOR_QUANT8_ASYMM: + case OperandType::TENSOR_QUANT8_SYMM: + case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: + return 0; + default: + return 0; + } +} + +static void mutateOperandRankTest(const std::shared_ptr& device, const Model& model) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + const uint32_t invalidRank = getInvalidRank(model.main.operands[operand].type); + if (invalidRank == 0) { + continue; + } + const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) + + " has rank of " + std::to_string(invalidRank); + validate(device, message, model, + [operand, invalidRank](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].dimensions = + std::vector(invalidRank, 0); + }); + } +} + +///////////////////////// VALIDATE OPERAND SCALE ///////////////////////// + +static float getInvalidScale(OperandType type) { + switch (type) { + case OperandType::FLOAT16: + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::BOOL: + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_FLOAT16: + case OperandType::TENSOR_FLOAT32: + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: + case OperandType::SUBGRAPH: + return 1.0f; + case OperandType::TENSOR_INT32: + return -1.0f; + case OperandType::TENSOR_QUANT8_SYMM: + case OperandType::TENSOR_QUANT8_ASYMM: + case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: + return 0.0f; + default: + return 0.0f; + } +} + +static void mutateOperandScaleTest(const std::shared_ptr& device, const Model& model) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + const float invalidScale = getInvalidScale(model.main.operands[operand].type); + const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) + + " has scale of " + std::to_string(invalidScale); + validate(device, message, model, + [operand, invalidScale](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].scale = invalidScale; + }); + } +} + +///////////////////////// VALIDATE OPERAND ZERO POINT ///////////////////////// + +static std::vector getInvalidZeroPoints(OperandType type) { + switch (type) { + case OperandType::FLOAT16: + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::BOOL: + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_FLOAT16: + case OperandType::TENSOR_FLOAT32: + case OperandType::TENSOR_INT32: + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: + case OperandType::SUBGRAPH: + return {1}; + case OperandType::TENSOR_QUANT8_ASYMM: + return {-1, 256}; + case OperandType::TENSOR_QUANT8_SYMM: + return {-129, -1, 1, 128}; + case OperandType::TENSOR_QUANT16_ASYMM: + return {-1, 65536}; + case OperandType::TENSOR_QUANT16_SYMM: + return {-32769, -1, 1, 32768}; + default: + return {}; + } +} + +static void mutateOperandZeroPointTest(const std::shared_ptr& device, const Model& model) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + const std::vector invalidZeroPoints = + getInvalidZeroPoints(model.main.operands[operand].type); + for (int32_t invalidZeroPoint : invalidZeroPoints) { + const std::string message = "mutateOperandZeroPointTest: operand " + + std::to_string(operand) + " has zero point of " + + std::to_string(invalidZeroPoint); + validate(device, message, model, + [operand, invalidZeroPoint](Model* model, ExecutionPreference*, Priority*) { + model->main.operands[operand].zeroPoint = invalidZeroPoint; + }); + } + } +} + +///////////////////////// VALIDATE OPERAND LIFETIME ///////////////////////////////////////////// + +static std::vector getInvalidLifeTimes(const Model& model, size_t modelSize, + const Operand& operand) { + // TODO: Support OperandLifeTime::CONSTANT_REFERENCE as an invalid lifetime + // TODO: Support OperandLifeTime::NO_VALUE as an invalid lifetime + + // Ways to get an invalid lifetime: + // - change whether a lifetime means an operand should have a writer + std::vector ret; + switch (operand.lifetime) { + case OperandLifeTime::SUBGRAPH_OUTPUT: + case OperandLifeTime::TEMPORARY_VARIABLE: + ret = { + OperandLifeTime::SUBGRAPH_INPUT, + OperandLifeTime::CONSTANT_COPY, + }; + break; + case OperandLifeTime::CONSTANT_COPY: + case OperandLifeTime::CONSTANT_POOL: + case OperandLifeTime::SUBGRAPH_INPUT: + ret = { + OperandLifeTime::TEMPORARY_VARIABLE, + OperandLifeTime::SUBGRAPH_OUTPUT, + }; + break; + case OperandLifeTime::NO_VALUE: + // Not enough information to know whether + // TEMPORARY_VARIABLE or CONSTANT_COPY would be invalid -- + // is this operand written (then CONSTANT_COPY would be + // invalid) or not (then TEMPORARY_VARIABLE would be + // invalid)? + break; + case OperandLifeTime::SUBGRAPH: + break; + default: + ADD_FAILURE(); + break; + } + + const size_t operandSize = sizeOfData(operand); // will be zero if shape is unknown + if (!operandSize || + exceedsBinderSizeLimit(modelSize + constantCopyExtraSize(model, operandSize))) { + // Unknown size or too-large size + ret.erase(std::remove(ret.begin(), ret.end(), OperandLifeTime::CONSTANT_COPY), ret.end()); + } + + return ret; +} + +static void mutateOperandLifeTimeTest(const std::shared_ptr& device, const Model& model) { + const size_t modelSize = sizeForBinder(model); + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + const std::vector invalidLifeTimes = + getInvalidLifeTimes(model, modelSize, model.main.operands[operand]); + for (OperandLifeTime invalidLifeTime : invalidLifeTimes) { + const std::string message = "mutateOperandLifetimeTest: operand " + + std::to_string(operand) + " has lifetime " + + toString(invalidLifeTime) + " instead of lifetime " + + toString(model.main.operands[operand].lifetime); + validate(device, message, model, + [operand, invalidLifeTime](Model* model, ExecutionPreference*, Priority*) { + static const DataLocation kZeroDataLocation = {}; + Operand& operandObj = model->main.operands[operand]; + switch (operandObj.lifetime) { + case OperandLifeTime::SUBGRAPH_INPUT: { + auto& inputs = model->main.inputIndexes; + inputs.erase(std::remove(inputs.begin(), inputs.end(), operand), + inputs.end()); + break; + } + case OperandLifeTime::SUBGRAPH_OUTPUT: { + auto& outputs = model->main.outputIndexes; + outputs.erase(std::remove(outputs.begin(), outputs.end(), operand), + outputs.end()); + break; + } + default: + break; + } + operandObj.lifetime = invalidLifeTime; + operandObj.location = kZeroDataLocation; + switch (invalidLifeTime) { + case OperandLifeTime::CONSTANT_COPY: { + becomeConstantCopy(model, &operandObj); + break; + } + case OperandLifeTime::SUBGRAPH_INPUT: + model->main.inputIndexes.push_back(operand); + break; + case OperandLifeTime::SUBGRAPH_OUTPUT: + model->main.outputIndexes.push_back(operand); + break; + default: + break; + } + }); + } + } +} + +///////////////////////// VALIDATE OPERAND INPUT-or-OUTPUT ////////////////////////////////////// + +static std::optional getInputOutputLifeTime(const Model& model, size_t modelSize, + const Operand& operand) { + // Ways to get an invalid lifetime (with respect to model inputIndexes and outputIndexes): + // - change whether a lifetime means an operand is a model input, a model output, or neither + // - preserve whether or not a lifetime means an operand should have a writer + switch (operand.lifetime) { + case OperandLifeTime::CONSTANT_COPY: + case OperandLifeTime::CONSTANT_POOL: + return OperandLifeTime::SUBGRAPH_INPUT; + case OperandLifeTime::SUBGRAPH_INPUT: { + const size_t operandSize = sizeOfData(operand); // will be zero if shape is unknown + if (!operandSize || + exceedsBinderSizeLimit(modelSize + constantCopyExtraSize(model, operandSize))) { + // Unknown size or too-large size + break; + } + return OperandLifeTime::CONSTANT_COPY; + } + case OperandLifeTime::SUBGRAPH_OUTPUT: + return OperandLifeTime::TEMPORARY_VARIABLE; + case OperandLifeTime::TEMPORARY_VARIABLE: + return OperandLifeTime::SUBGRAPH_OUTPUT; + case OperandLifeTime::NO_VALUE: + // Not enough information to know whether + // TEMPORARY_VARIABLE or CONSTANT_COPY would be an + // appropriate choice -- is this operand written (then + // TEMPORARY_VARIABLE would be appropriate) or not (then + // CONSTANT_COPY would be appropriate)? + break; + case OperandLifeTime::SUBGRAPH: + break; + default: + ADD_FAILURE(); + break; + } + + return std::nullopt; +} + +static void mutateOperandInputOutputTest(const std::shared_ptr& device, + const Model& model) { + const size_t modelSize = sizeForBinder(model); + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + const std::optional changedLifeTime = + getInputOutputLifeTime(model, modelSize, model.main.operands[operand]); + if (changedLifeTime) { + const std::string message = "mutateOperandInputOutputTest: operand " + + std::to_string(operand) + " has lifetime " + + toString(*changedLifeTime) + " instead of lifetime " + + toString(model.main.operands[operand].lifetime); + validate(device, message, model, + [operand, changedLifeTime](Model* model, ExecutionPreference*, Priority*) { + static const DataLocation kZeroDataLocation = {}; + Operand& operandObj = model->main.operands[operand]; + operandObj.lifetime = *changedLifeTime; + operandObj.location = kZeroDataLocation; + if (*changedLifeTime == OperandLifeTime::CONSTANT_COPY) { + becomeConstantCopy(model, &operandObj); + } + }); + } + } +} + +///////////////////////// VALIDATE OPERAND NUMBER OF WRITERS //////////////////////////////////// + +static void mutateOperandAddWriterTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + for (size_t badOutputNum = 0; + badOutputNum < model.main.operations[operation].outputs.size(); ++badOutputNum) { + const uint32_t outputOperandIndex = + model.main.operations[operation].outputs[badOutputNum]; + const std::string message = "mutateOperandAddWriterTest: operation " + + std::to_string(operation) + " writes to " + + std::to_string(outputOperandIndex); + // We'll insert a copy of the operation, all of whose + // OTHER output operands are newly-created -- i.e., + // there'll only be a duplicate write of ONE of that + // operation's output operands. + validate(device, message, model, + [operation, badOutputNum](Model* model, ExecutionPreference*, Priority*) { + Operation newOperation = model->main.operations[operation]; + for (size_t outputNum = 0; outputNum < newOperation.outputs.size(); + ++outputNum) { + if (outputNum == badOutputNum) continue; + + Operand operandValue = + model->main.operands[newOperation.outputs[outputNum]]; + if (operandValue.lifetime == OperandLifeTime::SUBGRAPH_OUTPUT) { + operandValue.lifetime = OperandLifeTime::TEMPORARY_VARIABLE; + } else { + ASSERT_EQ(operandValue.lifetime, + OperandLifeTime::TEMPORARY_VARIABLE); + } + newOperation.outputs[outputNum] = model->main.operands.size(); + model->main.operands.push_back(operandValue); + } + // Where do we insert the extra writer (a new + // operation)? It has to be later than all the + // writers of its inputs. The easiest thing to do + // is to insert it at the end of the operation + // sequence. + model->main.operations.push_back(newOperation); + }); + } + } +} + +///////////////////////// VALIDATE EXTRA ??? ///////////////////////// + +// TODO: Operand::location + +///////////////////////// VALIDATE OPERATION OPERAND TYPE ///////////////////////// + +static void mutateOperand(Operand* operand, OperandType type) { + Operand newOperand = *operand; + newOperand.type = type; + switch (type) { + case OperandType::FLOAT16: + case OperandType::FLOAT32: + case OperandType::INT32: + case OperandType::UINT32: + case OperandType::BOOL: + newOperand.dimensions = {}; + newOperand.scale = 0.0f; + newOperand.zeroPoint = 0; + break; + case OperandType::TENSOR_BOOL8: + case OperandType::TENSOR_FLOAT16: + case OperandType::TENSOR_FLOAT32: + newOperand.dimensions = operand->dimensions.size() > 0 ? operand->dimensions + : std::vector({1}); + newOperand.scale = 0.0f; + newOperand.zeroPoint = 0; + break; + case OperandType::TENSOR_INT32: + newOperand.dimensions = operand->dimensions.size() > 0 ? operand->dimensions + : std::vector({1}); + newOperand.zeroPoint = 0; + break; + case OperandType::TENSOR_QUANT8_ASYMM: + case OperandType::TENSOR_QUANT8_SYMM: + case OperandType::TENSOR_QUANT16_ASYMM: + case OperandType::TENSOR_QUANT16_SYMM: + newOperand.dimensions = operand->dimensions.size() > 0 ? operand->dimensions + : std::vector({1}); + newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f; + break; + case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: { + newOperand.dimensions = operand->dimensions.size() > 0 ? operand->dimensions + : std::vector({1}); + newOperand.scale = 0.0f; + newOperand.zeroPoint = 0; + + SymmPerChannelQuantParams channelQuant; + channelQuant.channelDim = 0; + channelQuant.scales = std::vector( + operand->dimensions.size() > 0 ? static_cast(operand->dimensions[0]) + : 0); + for (size_t i = 0; i < channelQuant.scales.size(); ++i) { + channelQuant.scales[i] = 1.0f; + } + newOperand.extraParams->set( + std::move(channelQuant)); + } break; + default: + break; + } + *operand = newOperand; +} + +static bool mutateOperationOperandTypeSkip(size_t operand, OperandType type, const Model& model) { + if (type == model.main.operands[operand].type) { + return true; + } + for (const Operation& operation : model.main.operations) { + // Skip mutateOperationOperandTypeTest for the following operations. + // - LSH_PROJECTION's second argument is allowed to have any type. + // - ARGMIN and ARGMAX's first argument can be any of + // TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM). + // - CAST's argument can be any of TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM). + // - RANDOM_MULTINOMIAL's argument can be either TENSOR_FLOAT16 or TENSOR_FLOAT32. + // - DEQUANTIZE input can be any of + // TENSOR_(QUANT8_ASYMM|QUANT8_ASYMM_SIGNED|QUANT8_SYMM|QUANT8_SYMM_PER_CHANNEL), + // output can be of either TENSOR_FLOAT16 or TENSOR_FLOAT32. + // - QUANTIZE input can be either TENSOR_FLOAT16 or TENSOR_FLOAT32 + // - CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL + // - DEPTHWISE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL + // - GROUPED_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL + // - TRANSPOSE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL + // - AXIS_ALIGNED_BBOX_TRANSFORM bounding boxes (arg 1) can be of + // TENSOR_QUANT8_ASYMM or TENSOR_QUANT8_ASYMM_SIGNED. + // - RANK's input can have any TENSOR_* type. + switch (operation.type) { + case OperationType::LSH_PROJECTION: { + if (operand == operation.inputs[1]) { + return true; + } + } break; + case OperationType::CAST: + case OperationType::ARGMAX: + case OperationType::ARGMIN: { + if (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32 || + type == OperandType::TENSOR_INT32 || type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED) { + return true; + } + } break; + case OperationType::QUANTIZE: { + if (operand == operation.inputs[0] && + (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) { + return true; + } + if (operand == operation.outputs[0] && + (type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)) { + return true; + } + } break; + case OperationType::RANDOM_MULTINOMIAL: { + if (operand == operation.inputs[0] && + (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) { + return true; + } + } break; + case OperationType::DEQUANTIZE: { + if (operand == operation.inputs[0] && + (type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED || + type == OperandType::TENSOR_QUANT8_SYMM || + type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) { + return true; + } + if (operand == operation.outputs[0] && + (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) { + return true; + } + } break; + case OperationType::TRANSPOSE_CONV_2D: + case OperationType::GROUPED_CONV_2D: + case OperationType::DEPTHWISE_CONV_2D: + case OperationType::CONV_2D: { + if (operand == operation.inputs[1] && + (type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) { + return true; + } + } break; + case OperationType::AXIS_ALIGNED_BBOX_TRANSFORM: { + if (operand == operation.inputs[1] && + (type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)) { + return true; + } + } break; + case OperationType::RANK: { + if (operand == operation.inputs[0] && + (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32 || + type == OperandType::TENSOR_INT32 || + type == OperandType::TENSOR_QUANT8_ASYMM || + type == OperandType::TENSOR_QUANT16_SYMM || + type == OperandType::TENSOR_BOOL8 || + type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL || + type == OperandType::TENSOR_QUANT16_ASYMM || + type == OperandType::TENSOR_QUANT8_SYMM || + type == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)) { + return true; + } + } break; + default: + break; + } + } + return false; +} + +static void mutateOperationOperandTypeTest(const std::shared_ptr& device, + const Model& model) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + for (OperandType invalidOperandType : ndk::enum_range()) { + if (mutateOperationOperandTypeSkip(operand, invalidOperandType, model)) { + continue; + } + const std::string message = "mutateOperationOperandTypeTest: operand " + + std::to_string(operand) + " set to type " + + toString(invalidOperandType); + validate(device, message, model, + [operand, invalidOperandType](Model* model, ExecutionPreference*, Priority*) { + mutateOperand(&model->main.operands[operand], invalidOperandType); + }); + } + } +} + +///////////////////////// VALIDATE MODEL OPERATION TYPE ///////////////////////// + +static const int32_t invalidOperationTypes[] = { + -1, + static_cast(*(ndk::enum_range().end() - 1)) + 1, +}; + +static void mutateOperationTypeTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + for (int32_t invalidOperationType : invalidOperationTypes) { + const std::string message = "mutateOperationTypeTest: operation " + + std::to_string(operation) + " set to value " + + std::to_string(invalidOperationType); + validate(device, message, model, + [operation, invalidOperationType](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].type = + static_cast(invalidOperationType); + }); + } + } +} + +///////////////////////// VALIDATE MODEL OPERATION INPUT OPERAND INDEX ///////////////////////// + +static void mutateOperationInputOperandIndexTest(const std::shared_ptr& device, + const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + const uint32_t invalidOperand = model.main.operands.size(); + for (size_t input = 0; input < model.main.operations[operation].inputs.size(); ++input) { + const std::string message = "mutateOperationInputOperandIndexTest: operation " + + std::to_string(operation) + " input " + + std::to_string(input); + validate(device, message, model, + [operation, input, invalidOperand](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].inputs[input] = invalidOperand; + }); + } + } +} + +///////////////////////// VALIDATE MODEL OPERATION OUTPUT OPERAND INDEX ///////////////////////// + +static void mutateOperationOutputOperandIndexTest(const std::shared_ptr& device, + const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + const uint32_t invalidOperand = model.main.operands.size(); + for (size_t output = 0; output < model.main.operations[operation].outputs.size(); + ++output) { + const std::string message = "mutateOperationOutputOperandIndexTest: operation " + + std::to_string(operation) + " output " + + std::to_string(output); + validate(device, message, model, + [operation, output, invalidOperand](Model* model, ExecutionPreference*, + Priority*) { + model->main.operations[operation].outputs[output] = invalidOperand; + }); + } + } +} + +///////////////////////// VALIDATE MODEL OPERANDS WRITTEN /////////////////////////////////////// + +static void mutateOperationRemoveWriteTest(const std::shared_ptr& device, + const Model& model, + const std::vector& numberOfConsumers) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + for (size_t outputNum = 0; outputNum < model.main.operations[operation].outputs.size(); + ++outputNum) { + const uint32_t outputOperandIndex = model.main.operations[operation].outputs[outputNum]; + if (numberOfConsumers[outputOperandIndex] > 0) { + const std::string message = "mutateOperationRemoveWriteTest: operation " + + std::to_string(operation) + " writes to " + + std::to_string(outputOperandIndex); + validate(device, message, model, + [operation, outputNum](Model* model, ExecutionPreference*, Priority*) { + int32_t& outputOperandIndex = + model->main.operations[operation].outputs[outputNum]; + Operand operandValue = model->main.operands[outputOperandIndex]; + if (operandValue.lifetime == OperandLifeTime::SUBGRAPH_OUTPUT) { + operandValue.lifetime = OperandLifeTime::TEMPORARY_VARIABLE; + } else { + ASSERT_EQ(operandValue.lifetime, + OperandLifeTime::TEMPORARY_VARIABLE); + } + outputOperandIndex = model->main.operands.size(); + model->main.operands.push_back(operandValue); + }); + } + } + } +} + +///////////////////////// REMOVE OPERAND FROM EVERYTHING ///////////////////////// + +static void removeValueAndDecrementGreaterValues(std::vector* vec, uint32_t value) { + if (vec) { + // remove elements matching "value" + vec->erase(std::remove(vec->begin(), vec->end(), value), vec->end()); + + // decrement elements exceeding "value" + std::transform(vec->begin(), vec->end(), vec->begin(), + [value](uint32_t v) { return v > value ? v-- : v; }); + } +} + +static void removeOperand(Model* model, uint32_t index) { + model->main.operands.erase(model->main.operands.begin() + index); + for (Operation& operation : model->main.operations) { + removeValueAndDecrementGreaterValues(&operation.inputs, index); + removeValueAndDecrementGreaterValues(&operation.outputs, index); + } + removeValueAndDecrementGreaterValues(&model->main.inputIndexes, index); + removeValueAndDecrementGreaterValues(&model->main.outputIndexes, index); +} + +static bool removeOperandSkip(size_t operandIndex, const Model& model, + const std::vector& numberOfConsumers) { + if (numberOfConsumers[operandIndex] == 0) { + // Removing an unused operand has no effect. + return true; + } + for (const Operation& operation : model.main.operations) { + // Skip removeOperandTest for the following operations. + // - SPLIT's outputs are not checked during prepareModel. + if (operation.type == OperationType::SPLIT) { + for (const size_t index : operation.outputs) { + if (index == operandIndex) { + return true; + } + } + } + // BIDIRECTIONAL_SEQUENCE_LSTM and BIDIRECTIONAL_SEQUENCE_RNN can have + // either one, two, three or four outputs depending on their + // mergeOutputs parameter and if state outputs are provided. + // UNIDIRECTIONAL_SEQUENCE_LSTM and UNIDIRECTIONAL_SEQUENCE_RNN can have + // either one or three outputs depending on whether state outputs are + // provided. + if (operation.type == OperationType::UNIDIRECTIONAL_SEQUENCE_LSTM || + operation.type == OperationType::UNIDIRECTIONAL_SEQUENCE_RNN || + operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_LSTM || + operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_RNN) { + for (const size_t index : operation.outputs) { + if (index == operandIndex) { + return true; + } + } + } + } + return false; +} + +static void removeOperandTest(const std::shared_ptr& device, const Model& model, + const std::vector& numberOfConsumers) { + for (size_t operand = 0; operand < model.main.operands.size(); ++operand) { + if (removeOperandSkip(operand, model, numberOfConsumers)) { + continue; + } + const std::string message = "removeOperandTest: operand " + std::to_string(operand); + validate(device, message, model, [operand](Model* model, ExecutionPreference*, Priority*) { + removeOperand(model, operand); + }); + } +} + +///////////////////////// REMOVE OPERATION ///////////////////////// + +static void removeOperation(Model* model, uint32_t index) { + auto& operations = model->main.operations; + operations.erase(operations.begin() + index); +} + +static void removeOperationTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + const std::string message = "removeOperationTest: operation " + std::to_string(operation); + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + removeOperation(model, operation); + }); + } +} + +///////////////////////// REMOVE OPERATION INPUT ///////////////////////// + +static bool removeOperationInputSkip(const Operation& op, size_t input) { + // Skip removeOperationInputTest for the following operations. + // - CONCATENATION has at least 2 inputs, with the last element being INT32. + // - CONV_2D, DEPTHWISE_CONV_2D, MAX_POOL_2D, AVERAGE_POOL_2D, L2_POOL_2D, RESIZE_BILINEAR, + // SPACE_TO_DEPTH, SPACE_TO_DEPTH, SPACE_TO_BATCH_ND, BATCH_TO_SPACE_ND can have an optional + // layout parameter. + // RESIZE_BILINEAR and RESIZE_NEAREST_NEIGHBOR can have optional + // align_corners and half_pixel_centers parameters. + // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis + // parameter. + switch (op.type) { + case OperationType::CONCATENATION: { + if (op.inputs.size() > 2 && input != op.inputs.size() - 1) { + return true; + } + } break; + case OperationType::DEPTHWISE_CONV_2D: { + if ((op.inputs.size() == 12 && input == 11) || (op.inputs.size() == 9 && input == 8)) { + return true; + } + } break; + case OperationType::CONV_2D: + case OperationType::AVERAGE_POOL_2D: + case OperationType::MAX_POOL_2D: + case OperationType::L2_POOL_2D: { + if ((op.inputs.size() == 11 && input == 10) || (op.inputs.size() == 8 && input == 7)) { + return true; + } + } break; + case OperationType::RESIZE_BILINEAR: { + if (op.inputs.size() >= 4 && input >= 3) { + return true; + } + } break; + case OperationType::RESIZE_NEAREST_NEIGHBOR: { + if (op.inputs.size() >= 5 && input >= 3) { + return true; + } + } break; + case OperationType::SPACE_TO_DEPTH: + case OperationType::DEPTH_TO_SPACE: + case OperationType::BATCH_TO_SPACE_ND: { + if (op.inputs.size() == 3 && input == 2) { + return true; + } + } break; + case OperationType::SPACE_TO_BATCH_ND: { + if (op.inputs.size() == 4 && input == 3) { + return true; + } + } break; + case OperationType::L2_NORMALIZATION: { + if (op.inputs.size() == 2 && input == 1) { + return true; + } + } break; + case OperationType::LOCAL_RESPONSE_NORMALIZATION: { + if (op.inputs.size() == 6 && input == 5) { + return true; + } + } break; + case OperationType::SOFTMAX: { + if (op.inputs.size() == 3 && input == 2) { + return true; + } + } break; + default: + break; + } + return false; +} + +static void removeOperationInputTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + for (size_t input = 0; input < model.main.operations[operation].inputs.size(); ++input) { + const Operation& op = model.main.operations[operation]; + if (removeOperationInputSkip(op, input)) { + continue; + } + const std::string message = "removeOperationInputTest: operation " + + std::to_string(operation) + ", input " + + std::to_string(input); + validate(device, message, model, + [operation, input](Model* model, ExecutionPreference*, Priority*) { + auto& inputs = model->main.operations[operation].inputs; + inputs.erase(inputs.begin() + input); + }); + } + } +} + +///////////////////////// REMOVE OPERATION OUTPUT ///////////////////////// + +static void removeOperationOutputTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + for (size_t output = 0; output < model.main.operations[operation].outputs.size(); + ++output) { + const std::string message = "removeOperationOutputTest: operation " + + std::to_string(operation) + ", output " + + std::to_string(output); + validate(device, message, model, + [operation, output](Model* model, ExecutionPreference*, Priority*) { + auto& outputs = model->main.operations[operation].outputs; + outputs.erase(outputs.begin() + output); + }); + } + } +} + +///////////////////////// MODEL VALIDATION ///////////////////////// + +// TODO: remove model input +// TODO: remove model output +// TODO: add unused operation + +///////////////////////// ADD OPERATION INPUT ///////////////////////// + +static bool addOperationInputSkip(const Operation& op) { + // Skip addOperationInputTest for the following operations. + // - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional INT32 axis + // parameter. + if ((op.type == OperationType::L2_NORMALIZATION && op.inputs.size() == 1) || + (op.type == OperationType::LOCAL_RESPONSE_NORMALIZATION && op.inputs.size() == 5) || + (op.type == OperationType::SOFTMAX && op.inputs.size() == 2) || + (op.type == OperationType::RESIZE_BILINEAR && op.inputs.size() < 6) || + (op.type == OperationType::RESIZE_NEAREST_NEIGHBOR && op.inputs.size() < 6)) { + return true; + } + return false; +} + +static void addOperationInputTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + if (addOperationInputSkip(model.main.operations[operation])) { + continue; + } + const std::string message = "addOperationInputTest: operation " + std::to_string(operation); + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_INPUT); + model->main.operations[operation].inputs.push_back(index); + model->main.inputIndexes.push_back(index); + }); + } +} + +///////////////////////// ADD OPERATION OUTPUT ///////////////////////// + +static void addOperationOutputTest(const std::shared_ptr& device, const Model& model) { + for (size_t operation = 0; operation < model.main.operations.size(); ++operation) { + const std::string message = + "addOperationOutputTest: operation " + std::to_string(operation); + validate(device, message, model, + [operation](Model* model, ExecutionPreference*, Priority*) { + uint32_t index = addOperand(model, OperandLifeTime::SUBGRAPH_OUTPUT); + model->main.operations[operation].outputs.push_back(index); + model->main.outputIndexes.push_back(index); + }); + } +} + +///////////////////////// VALIDATE EXECUTION PREFERENCE ///////////////////////// + +static const int32_t invalidExecutionPreferences[] = { + static_cast(ExecutionPreference::LOW_POWER) - 1, // lower bound + static_cast(ExecutionPreference::SUSTAINED_SPEED) + 1, // upper bound +}; + +static void mutateExecutionPreferenceTest(const std::shared_ptr& device, + const Model& model) { + for (int32_t invalidPreference : invalidExecutionPreferences) { + const std::string message = + "mutateExecutionPreferenceTest: preference " + std::to_string(invalidPreference); + validate(device, message, model, + [invalidPreference](Model*, ExecutionPreference* preference, Priority*) { + *preference = static_cast(invalidPreference); + }); + } +} + +///////////////////////// VALIDATE PRIORITY ///////////////////////// + +static const int32_t invalidPriorities[] = { + static_cast(Priority::LOW) - 1, // lower bound + static_cast(Priority::HIGH) + 1, // upper bound +}; + +static void mutateExecutionPriorityTest(const std::shared_ptr& device, + const Model& model) { + for (int32_t invalidPriority : invalidPriorities) { + const std::string message = + "mutatePriorityTest: priority " + std::to_string(invalidPriority); + validate(device, message, model, + [invalidPriority](Model*, ExecutionPreference*, Priority* priority) { + *priority = static_cast(invalidPriority); + }); + } +} + +////////////////////////// ENTRY POINT ////////////////////////////// + +void validateModel(const std::shared_ptr& device, const Model& model) { + const auto numberOfConsumers = nn::countNumberOfConsumers( + model.main.operands.size(), nn::convert(model.main.operations).value()); + mutateExecutionOrderTest(device, model, numberOfConsumers); + mutateOperandTypeTest(device, model); + mutateOperandRankTest(device, model); + mutateOperandScaleTest(device, model); + mutateOperandZeroPointTest(device, model); + mutateOperandLifeTimeTest(device, model); + mutateOperandInputOutputTest(device, model); + mutateOperandAddWriterTest(device, model); + mutateOperationOperandTypeTest(device, model); + mutateOperationTypeTest(device, model); + mutateOperationInputOperandIndexTest(device, model); + mutateOperationOutputOperandIndexTest(device, model); + mutateOperationRemoveWriteTest(device, model, numberOfConsumers); + removeOperandTest(device, model, numberOfConsumers); + removeOperationTest(device, model); + removeOperationInputTest(device, model); + removeOperationOutputTest(device, model); + addOperationInputTest(device, model); + addOperationOutputTest(device, model); + mutateExecutionPreferenceTest(device, model); + mutateExecutionPriorityTest(device, model); +} + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/ValidateRequest.cpp b/neuralnetworks/aidl/vts/functional/ValidateRequest.cpp new file mode 100644 index 0000000000..db8f429f13 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/ValidateRequest.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" + +#include + +#include + +#include +#include + +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "Utils.h" +#include "VtsHalNeuralnetworks.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using ExecutionMutation = std::function; + +///////////////////////// UTILITY FUNCTIONS ///////////////////////// + +// Primary validation function. This function will take a valid request, apply a +// mutation to it to invalidate the request, then pass it to interface calls +// that use the request. +static void validate(const std::shared_ptr& preparedModel, + const std::string& message, const Request& originalRequest, + const ExecutionMutation& mutate) { + Request request = utils::clone(originalRequest).value(); + mutate(&request); + + // We'd like to test both with timing requested and without timing + // requested. Rather than running each test both ways, we'll decide whether + // to request timing by hashing the message. We do not use std::hash because + // it is not guaranteed stable across executions. + char hash = 0; + for (auto c : message) { + hash ^= c; + }; + bool measure = (hash & 1); + + // synchronous + { + SCOPED_TRACE(message + " [executeSynchronously]"); + ExecutionResult executionResult; + const auto executeStatus = preparedModel->executeSynchronously( + request, measure, kNoDeadline, kOmittedTimeoutDuration, &executionResult); + ASSERT_FALSE(executeStatus.isOk()); + ASSERT_EQ(executeStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(executeStatus.getServiceSpecificError()), + ErrorStatus::INVALID_ARGUMENT); + } + + // fenced + { + SCOPED_TRACE(message + " [executeFenced]"); + ndk::ScopedFileDescriptor syncFence; + std::shared_ptr callback; + const auto executeStatus = preparedModel->executeFenced(request, {}, false, kNoDeadline, + kOmittedTimeoutDuration, + kNoDuration, &syncFence, &callback); + ASSERT_FALSE(executeStatus.isOk()); + ASSERT_EQ(executeStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_EQ(static_cast(executeStatus.getServiceSpecificError()), + ErrorStatus::INVALID_ARGUMENT); + } +} + +///////////////////////// REMOVE INPUT //////////////////////////////////// + +static void removeInputTest(const std::shared_ptr& preparedModel, + const Request& request) { + for (size_t input = 0; input < request.inputs.size(); ++input) { + const std::string message = "removeInput: removed input " + std::to_string(input); + validate(preparedModel, message, request, [input](Request* request) { + request->inputs.erase(request->inputs.begin() + input); + }); + } +} + +///////////////////////// REMOVE OUTPUT //////////////////////////////////// + +static void removeOutputTest(const std::shared_ptr& preparedModel, + const Request& request) { + for (size_t output = 0; output < request.outputs.size(); ++output) { + const std::string message = "removeOutput: removed Output " + std::to_string(output); + validate(preparedModel, message, request, [output](Request* request) { + request->outputs.erase(request->outputs.begin() + output); + }); + } +} + +///////////////////////////// ENTRY POINT ////////////////////////////////// + +void validateRequest(const std::shared_ptr& preparedModel, const Request& request) { + removeInputTest(preparedModel, request); + removeOutputTest(preparedModel, request); +} + +void validateRequestFailure(const std::shared_ptr& preparedModel, + const Request& request) { + SCOPED_TRACE("Expecting request to fail [executeSynchronously]"); + ExecutionResult executionResult; + const auto executeStatus = preparedModel->executeSynchronously( + request, false, kNoDeadline, kOmittedTimeoutDuration, &executionResult); + + ASSERT_FALSE(executeStatus.isOk()); + ASSERT_EQ(executeStatus.getExceptionCode(), EX_SERVICE_SPECIFIC); + ASSERT_NE(static_cast(executeStatus.getServiceSpecificError()), ErrorStatus::NONE); +} + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.cpp b/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.cpp new file mode 100644 index 0000000000..2d91b8edd9 --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "neuralnetworks_aidl_hal_test" +#include "VtsHalNeuralnetworks.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Callbacks.h" +#include "GeneratedTestHarness.h" +#include "Utils.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using implementation::PreparedModelCallback; + +// internal helper function +void createPreparedModel(const std::shared_ptr& device, const Model& model, + std::shared_ptr* preparedModel, bool reportSkipping) { + ASSERT_NE(nullptr, preparedModel); + *preparedModel = nullptr; + + // see if service can handle model + std::vector supportedOperations; + const auto supportedCallStatus = device->getSupportedOperations(model, &supportedOperations); + ASSERT_TRUE(supportedCallStatus.isOk()); + ASSERT_NE(0ul, supportedOperations.size()); + const bool fullySupportsModel = std::all_of( + supportedOperations.begin(), supportedOperations.end(), [](bool v) { return v; }); + + // launch prepare model + const std::shared_ptr preparedModelCallback = + ndk::SharedRefBase::make(); + const auto prepareLaunchStatus = + device->prepareModel(model, ExecutionPreference::FAST_SINGLE_ANSWER, kDefaultPriority, + kNoDeadline, {}, {}, kEmptyCacheToken, preparedModelCallback); + ASSERT_TRUE(prepareLaunchStatus.isOk()) << prepareLaunchStatus.getDescription(); + + // retrieve prepared model + preparedModelCallback->wait(); + const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); + *preparedModel = preparedModelCallback->getPreparedModel(); + + // The getSupportedOperations call returns a list of operations that are guaranteed not to fail + // if prepareModel is called, and 'fullySupportsModel' is true i.f.f. the entire model is + // guaranteed. If a driver has any doubt that it can prepare an operation, it must return false. + // So here, if a driver isn't sure if it can support an operation, but reports that it + // successfully prepared the model, the test can continue. + if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { + ASSERT_EQ(nullptr, preparedModel->get()); + if (!reportSkipping) { + return; + } + LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare " + "model that it does not support."; + std::cout << "[ ] Early termination of test because vendor service cannot " + "prepare model that it does not support." + << std::endl; + GTEST_SKIP(); + } + + ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus); + ASSERT_NE(nullptr, preparedModel->get()); +} + +void NeuralNetworksAidlTest::SetUp() { + testing::TestWithParam::SetUp(); + ASSERT_NE(kDevice, nullptr); +} + +static NamedDevice makeNamedDevice(const std::string& name) { + ndk::SpAIBinder binder(AServiceManager_getService(name.c_str())); + return {name, IDevice::fromBinder(binder)}; +} + +static std::vector getNamedDevicesImpl() { + // Retrieves the name of all service instances that implement IDevice, + // including any Lazy HAL instances. + const std::vector names = ::android::getAidlHalInstanceNames(IDevice::descriptor); + + // Get a handle to each device and pair it with its name. + std::vector namedDevices; + namedDevices.reserve(names.size()); + std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice); + return namedDevices; +} + +const std::vector& getNamedDevices() { + const static std::vector devices = getNamedDevicesImpl(); + return devices; +} + +std::string printNeuralNetworksAidlTest( + const testing::TestParamInfo& info) { + return gtestCompliantName(getName(info.param)); +} + +INSTANTIATE_DEVICE_TEST(NeuralNetworksAidlTest); + +// Forward declaration from ValidateModel.cpp +void validateModel(const std::shared_ptr& device, const Model& model); +// Forward declaration from ValidateRequest.cpp +void validateRequest(const std::shared_ptr& preparedModel, const Request& request); +// Forward declaration from ValidateRequest.cpp +void validateRequestFailure(const std::shared_ptr& preparedModel, + const Request& request); + +void validateEverything(const std::shared_ptr& device, const Model& model, + const Request& request) { + validateModel(device, model); + + // Create IPreparedModel. + std::shared_ptr preparedModel; + createPreparedModel(device, model, &preparedModel); + if (preparedModel == nullptr) return; + + validateRequest(preparedModel, request); + // HIDL also had test that expected executeFenced to fail on received null fd (-1). This is not + // allowed in AIDL and will result in EX_TRANSACTION_FAILED. +} + +void validateFailure(const std::shared_ptr& device, const Model& model, + const Request& request) { + // TODO: Should this always succeed? + // What if the invalid input is part of the model (i.e., a parameter). + validateModel(device, model); + + // Create IPreparedModel. + std::shared_ptr preparedModel; + createPreparedModel(device, model, &preparedModel); + if (preparedModel == nullptr) return; + + validateRequestFailure(preparedModel, request); +} + +TEST_P(ValidationTest, Test) { + const Model model = createModel(kTestModel); + ExecutionContext context; + const Request request = context.createRequest(kTestModel); + if (kTestModel.expectFailure) { + validateFailure(kDevice, model, request); + } else { + validateEverything(kDevice, model, request); + } +} + +INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) { + // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs" + // generated tests. + return testName.find("inputs_as_internal") == std::string::npos && + testName.find("all_tensors_as_inputs") == std::string::npos; +}); + +std::string toString(Executor executor) { + switch (executor) { + case Executor::ASYNC: + return "ASYNC"; + case Executor::SYNC: + return "SYNC"; + case Executor::BURST: + return "BURST"; + case Executor::FENCED: + return "FENCED"; + default: + CHECK(false); + } +} + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional diff --git a/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.h b/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.h new file mode 100644 index 0000000000..9b81ee116e --- /dev/null +++ b/neuralnetworks/aidl/vts/functional/VtsHalNeuralnetworks.h @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_NEURALNETWORKS_AIDL_VTS_HAL_NEURALNETWORKS_H +#define ANDROID_HARDWARE_NEURALNETWORKS_AIDL_VTS_HAL_NEURALNETWORKS_H + +#include +#include + +#include + +#include "Callbacks.h" +#include "Utils.h" + +namespace aidl::android::hardware::neuralnetworks::vts::functional { + +using NamedDevice = Named>; +using NeuralNetworksAidlTestParam = NamedDevice; + +class NeuralNetworksAidlTest : public testing::TestWithParam { + protected: + void SetUp() override; + const std::shared_ptr kDevice = getData(GetParam()); +}; + +const std::vector& getNamedDevices(); + +std::string printNeuralNetworksAidlTest( + const testing::TestParamInfo& info); + +#define INSTANTIATE_DEVICE_TEST(TestSuite) \ + GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(TestSuite); \ + INSTANTIATE_TEST_SUITE_P(PerInstance, TestSuite, testing::ValuesIn(getNamedDevices()), \ + printNeuralNetworksAidlTest) + +// Create an IPreparedModel object. If the model cannot be prepared, +// "preparedModel" will be nullptr instead. +void createPreparedModel(const std::shared_ptr& device, const Model& model, + std::shared_ptr* preparedModel, + bool reportSkipping = true); + +enum class Executor { ASYNC, SYNC, BURST, FENCED }; + +std::string toString(Executor executor); + +} // namespace aidl::android::hardware::neuralnetworks::vts::functional + +#endif // ANDROID_HARDWARE_NEURALNETWORKS_AIDL_VTS_HAL_NEURALNETWORKS_H