Copy VTS tests from v1.2 to v1.3
So that it's easier to see what actually has changed in VTS tests for
version 1.3
Bug: 139120468
Test: m
Change-Id: I09797f5f3898501a008186a22dd411b00e9e2c67
Merged-In: I09797f5f3898501a008186a22dd411b00e9e2c67
(cherry picked from commit 3b13b55ac1)
This commit is contained in:
Lev Proleev
parent
ce8c632a09
commit
13fdfcd44f
13 changed files with 4095 additions and 0 deletions
16
neuralnetworks/1.3/vts/OWNERS
Normal file
16
neuralnetworks/1.3/vts/OWNERS
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# Neuralnetworks team
|
||||
butlermichael@google.com
|
||||
dgross@google.com
|
||||
jeanluc@google.com
|
||||
levp@google.com
|
||||
miaowang@google.com
|
||||
mikie@google.com
|
||||
mks@google.com
|
||||
pszczepaniak@google.com
|
||||
slavash@google.com
|
||||
vddang@google.com
|
||||
xusongw@google.com
|
||||
|
||||
# VTS team
|
||||
yim@google.com
|
||||
yuexima@google.com
|
||||
114
neuralnetworks/1.3/vts/functional/BasicTests.cpp
Normal file
114
neuralnetworks/1.3/vts/functional/BasicTests.cpp
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "neuralnetworks_hidl_hal_test"
|
||||
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using V1_0::DeviceStatus;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::PerformanceInfo;
|
||||
|
||||
// create device test
|
||||
TEST_P(NeuralnetworksHidlTest, CreateDevice) {}
|
||||
|
||||
// status test
|
||||
TEST_P(NeuralnetworksHidlTest, StatusTest) {
|
||||
Return<DeviceStatus> status = kDevice->getStatus();
|
||||
ASSERT_TRUE(status.isOk());
|
||||
EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
|
||||
}
|
||||
|
||||
// initialization
|
||||
TEST_P(NeuralnetworksHidlTest, GetCapabilitiesTest) {
|
||||
using OperandPerformance = Capabilities::OperandPerformance;
|
||||
Return<void> ret = kDevice->getCapabilities_1_2([](ErrorStatus status,
|
||||
const Capabilities& capabilities) {
|
||||
EXPECT_EQ(ErrorStatus::NONE, status);
|
||||
|
||||
auto isPositive = [](const PerformanceInfo& perf) {
|
||||
return perf.execTime > 0.0f && perf.powerUsage > 0.0f;
|
||||
};
|
||||
|
||||
EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceScalar));
|
||||
EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceTensor));
|
||||
const auto& opPerf = capabilities.operandPerformance;
|
||||
EXPECT_TRUE(std::all_of(
|
||||
opPerf.begin(), opPerf.end(),
|
||||
[isPositive](const OperandPerformance& a) { return isPositive(a.info); }));
|
||||
EXPECT_TRUE(std::is_sorted(opPerf.begin(), opPerf.end(),
|
||||
[](const OperandPerformance& a, const OperandPerformance& b) {
|
||||
return a.type < b.type;
|
||||
}));
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
|
||||
// device version test
|
||||
TEST_P(NeuralnetworksHidlTest, GetDeviceVersionStringTest) {
|
||||
Return<void> ret =
|
||||
kDevice->getVersionString([](ErrorStatus status, const hidl_string& version) {
|
||||
EXPECT_EQ(ErrorStatus::NONE, status);
|
||||
EXPECT_LT(0, version.size());
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
|
||||
// device type test
|
||||
TEST_P(NeuralnetworksHidlTest, GetDeviceTypeTest) {
|
||||
Return<void> ret = kDevice->getType([](ErrorStatus status, DeviceType type) {
|
||||
EXPECT_EQ(ErrorStatus::NONE, status);
|
||||
EXPECT_TRUE(type == DeviceType::OTHER || type == DeviceType::CPU ||
|
||||
type == DeviceType::GPU || type == DeviceType::ACCELERATOR);
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
|
||||
// device supported extensions test
|
||||
TEST_P(NeuralnetworksHidlTest, GetDeviceSupportedExtensionsTest) {
|
||||
Return<void> ret = kDevice->getSupportedExtensions(
|
||||
[](ErrorStatus status, const hidl_vec<Extension>& extensions) {
|
||||
EXPECT_EQ(ErrorStatus::NONE, status);
|
||||
for (auto& extension : extensions) {
|
||||
std::string extensionName = extension.name;
|
||||
EXPECT_FALSE(extensionName.empty());
|
||||
for (char c : extensionName) {
|
||||
EXPECT_TRUE(('a' <= c && c <= 'z') || ('0' <= c && c <= '9') || c == '_' ||
|
||||
c == '.')
|
||||
<< "Extension name contains an illegal character: " << c;
|
||||
}
|
||||
EXPECT_NE(extensionName.find('.'), std::string::npos)
|
||||
<< "Extension name must start with the reverse domain name of the "
|
||||
"vendor";
|
||||
}
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
|
||||
// getNumberOfCacheFilesNeeded test
|
||||
TEST_P(NeuralnetworksHidlTest, getNumberOfCacheFilesNeeded) {
|
||||
Return<void> ret = kDevice->getNumberOfCacheFilesNeeded(
|
||||
[](ErrorStatus status, uint32_t numModelCache, uint32_t numDataCache) {
|
||||
EXPECT_EQ(ErrorStatus::NONE, status);
|
||||
EXPECT_LE(numModelCache,
|
||||
static_cast<uint32_t>(Constant::MAX_NUMBER_OF_CACHE_FILES));
|
||||
EXPECT_LE(numDataCache, static_cast<uint32_t>(Constant::MAX_NUMBER_OF_CACHE_FILES));
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
143
neuralnetworks/1.3/vts/functional/Callbacks.cpp
Normal file
143
neuralnetworks/1.3/vts/functional/Callbacks.cpp
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "Callbacks"
|
||||
|
||||
#include "1.2/Callbacks.h"
|
||||
|
||||
#include <android-base/logging.h>
|
||||
|
||||
#include <limits>
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::implementation {
|
||||
|
||||
using V1_0::ErrorStatus;
|
||||
|
||||
constexpr Timing kNoTiming = {.timeOnDevice = std::numeric_limits<uint64_t>::max(),
|
||||
.timeInDriver = std::numeric_limits<uint64_t>::max()};
|
||||
|
||||
// PreparedModelCallback methods begin here
|
||||
|
||||
Return<void> PreparedModelCallback::notify(ErrorStatus errorStatus,
|
||||
const sp<V1_0::IPreparedModel>& preparedModel) {
|
||||
{
|
||||
std::lock_guard<std::mutex> hold(mMutex);
|
||||
|
||||
// quick-return if object has already been notified
|
||||
if (mNotified) {
|
||||
return Void();
|
||||
}
|
||||
|
||||
// store results and mark as notified
|
||||
mErrorStatus = errorStatus;
|
||||
mPreparedModel = preparedModel;
|
||||
mNotified = true;
|
||||
}
|
||||
|
||||
mCondition.notify_all();
|
||||
return Void();
|
||||
}
|
||||
|
||||
Return<void> PreparedModelCallback::notify_1_2(ErrorStatus errorStatus,
|
||||
const sp<V1_2::IPreparedModel>& preparedModel) {
|
||||
return notify(errorStatus, preparedModel);
|
||||
}
|
||||
|
||||
void PreparedModelCallback::wait() const {
|
||||
std::unique_lock<std::mutex> lock(mMutex);
|
||||
mCondition.wait(lock, [this] { return mNotified; });
|
||||
}
|
||||
|
||||
ErrorStatus PreparedModelCallback::getStatus() const {
|
||||
wait();
|
||||
return mErrorStatus;
|
||||
}
|
||||
|
||||
sp<V1_0::IPreparedModel> PreparedModelCallback::getPreparedModel() const {
|
||||
wait();
|
||||
return mPreparedModel;
|
||||
}
|
||||
|
||||
// ExecutionCallback methods begin here
|
||||
|
||||
Return<void> ExecutionCallback::notify(ErrorStatus errorStatus) {
|
||||
notifyInternal(errorStatus, {}, kNoTiming);
|
||||
return Void();
|
||||
}
|
||||
|
||||
Return<void> ExecutionCallback::notify_1_2(ErrorStatus errorStatus,
|
||||
const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing) {
|
||||
if (errorStatus == ErrorStatus::OUTPUT_INSUFFICIENT_SIZE) {
|
||||
// outputShapes must not be empty if OUTPUT_INSUFFICIENT_SIZE.
|
||||
if (outputShapes.size() == 0) {
|
||||
LOG(ERROR) << "Notified with empty output shape vector when OUTPUT_INSUFFICIENT_SIZE";
|
||||
notifyInternal(ErrorStatus::GENERAL_FAILURE, {}, kNoTiming);
|
||||
return Void();
|
||||
}
|
||||
} else if (errorStatus != ErrorStatus::NONE) {
|
||||
// outputShapes must be empty if errorStatus is neither NONE nor OUTPUT_INSUFFICIENT_SIZE.
|
||||
if (outputShapes.size() != 0) {
|
||||
LOG(ERROR) << "Notified with non-empty output shape vector when error status is "
|
||||
"neither NONE nor OUTPUT_INSUFFICIENT_SIZE";
|
||||
notifyInternal(ErrorStatus::GENERAL_FAILURE, {}, kNoTiming);
|
||||
return Void();
|
||||
}
|
||||
}
|
||||
notifyInternal(errorStatus, outputShapes, timing);
|
||||
return Void();
|
||||
}
|
||||
|
||||
void ExecutionCallback::wait() const {
|
||||
std::unique_lock<std::mutex> lock(mMutex);
|
||||
mCondition.wait(lock, [this] { return mNotified; });
|
||||
}
|
||||
|
||||
ErrorStatus ExecutionCallback::getStatus() const {
|
||||
wait();
|
||||
return mErrorStatus;
|
||||
}
|
||||
|
||||
const std::vector<OutputShape>& ExecutionCallback::getOutputShapes() const {
|
||||
wait();
|
||||
return mOutputShapes;
|
||||
}
|
||||
|
||||
Timing ExecutionCallback::getTiming() const {
|
||||
wait();
|
||||
return mTiming;
|
||||
}
|
||||
|
||||
void ExecutionCallback::notifyInternal(ErrorStatus errorStatus,
|
||||
const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing) {
|
||||
{
|
||||
std::lock_guard<std::mutex> hold(mMutex);
|
||||
|
||||
// quick-return if object has already been notified
|
||||
if (mNotified) {
|
||||
return;
|
||||
}
|
||||
|
||||
mErrorStatus = errorStatus;
|
||||
mOutputShapes = outputShapes;
|
||||
mTiming = timing;
|
||||
mNotified = true;
|
||||
}
|
||||
mCondition.notify_all();
|
||||
}
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::implementation
|
||||
1374
neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp
Normal file
1374
neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp
Normal file
File diff suppressed because it is too large
Load diff
408
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
Normal file
408
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
Normal file
|
|
@ -0,0 +1,408 @@
|
|||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "GeneratedTestHarness.h"
|
||||
|
||||
#include <android-base/logging.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IDevice.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/types.h>
|
||||
#include <android/hardware/neuralnetworks/1.1/IDevice.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IDevice.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
|
||||
#include <android/hidl/allocator/1.0/IAllocator.h>
|
||||
#include <android/hidl/memory/1.0/IMemory.h>
|
||||
#include <hidlmemory/mapping.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
|
||||
#include "1.0/Utils.h"
|
||||
#include "1.2/Callbacks.h"
|
||||
#include "ExecutionBurstController.h"
|
||||
#include "MemoryUtils.h"
|
||||
#include "TestHarness.h"
|
||||
#include "Utils.h"
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using namespace test_helper;
|
||||
using hidl::memory::V1_0::IMemory;
|
||||
using implementation::ExecutionCallback;
|
||||
using implementation::PreparedModelCallback;
|
||||
using V1_0::DataLocation;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::OperandLifeTime;
|
||||
using V1_0::Request;
|
||||
using V1_1::ExecutionPreference;
|
||||
using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
|
||||
|
||||
enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
|
||||
|
||||
Model createModel(const TestModel& testModel) {
|
||||
// Model operands.
|
||||
hidl_vec<Operand> operands(testModel.operands.size());
|
||||
size_t constCopySize = 0, constRefSize = 0;
|
||||
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
|
||||
const auto& op = testModel.operands[i];
|
||||
|
||||
DataLocation loc = {};
|
||||
if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
|
||||
loc = {.poolIndex = 0,
|
||||
.offset = static_cast<uint32_t>(constCopySize),
|
||||
.length = static_cast<uint32_t>(op.data.size())};
|
||||
constCopySize += op.data.alignedSize();
|
||||
} else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
|
||||
loc = {.poolIndex = 0,
|
||||
.offset = static_cast<uint32_t>(constRefSize),
|
||||
.length = static_cast<uint32_t>(op.data.size())};
|
||||
constRefSize += op.data.alignedSize();
|
||||
}
|
||||
|
||||
Operand::ExtraParams extraParams;
|
||||
if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
|
||||
extraParams.channelQuant(SymmPerChannelQuantParams{
|
||||
.scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
|
||||
}
|
||||
|
||||
operands[i] = {.type = static_cast<OperandType>(op.type),
|
||||
.dimensions = op.dimensions,
|
||||
.numberOfConsumers = op.numberOfConsumers,
|
||||
.scale = op.scale,
|
||||
.zeroPoint = op.zeroPoint,
|
||||
.lifetime = static_cast<OperandLifeTime>(op.lifetime),
|
||||
.location = loc,
|
||||
.extraParams = std::move(extraParams)};
|
||||
}
|
||||
|
||||
// Model operations.
|
||||
hidl_vec<Operation> operations(testModel.operations.size());
|
||||
std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
|
||||
[](const TestOperation& op) -> Operation {
|
||||
return {.type = static_cast<OperationType>(op.type),
|
||||
.inputs = op.inputs,
|
||||
.outputs = op.outputs};
|
||||
});
|
||||
|
||||
// Constant copies.
|
||||
hidl_vec<uint8_t> operandValues(constCopySize);
|
||||
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
|
||||
const auto& op = testModel.operands[i];
|
||||
if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
|
||||
const uint8_t* begin = op.data.get<uint8_t>();
|
||||
const uint8_t* end = begin + op.data.size();
|
||||
std::copy(begin, end, operandValues.data() + operands[i].location.offset);
|
||||
}
|
||||
}
|
||||
|
||||
// Shared memory.
|
||||
hidl_vec<hidl_memory> pools = {};
|
||||
if (constRefSize > 0) {
|
||||
hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
|
||||
CHECK_NE(pools[0].size(), 0u);
|
||||
|
||||
// load data
|
||||
sp<IMemory> mappedMemory = mapMemory(pools[0]);
|
||||
CHECK(mappedMemory.get() != nullptr);
|
||||
uint8_t* mappedPtr =
|
||||
reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
|
||||
CHECK(mappedPtr != nullptr);
|
||||
|
||||
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
|
||||
const auto& op = testModel.operands[i];
|
||||
if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
|
||||
const uint8_t* begin = op.data.get<uint8_t>();
|
||||
const uint8_t* end = begin + op.data.size();
|
||||
std::copy(begin, end, mappedPtr + operands[i].location.offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {.operands = std::move(operands),
|
||||
.operations = std::move(operations),
|
||||
.inputIndexes = testModel.inputIndexes,
|
||||
.outputIndexes = testModel.outputIndexes,
|
||||
.operandValues = std::move(operandValues),
|
||||
.pools = std::move(pools),
|
||||
.relaxComputationFloat32toFloat16 = testModel.isRelaxed};
|
||||
}
|
||||
|
||||
static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
|
||||
const auto byteSize = testModel.operands[testModel.outputIndexes[index]].data.size();
|
||||
return byteSize > 1u;
|
||||
}
|
||||
|
||||
static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
|
||||
auto& length = request->outputs[outputIndex].location.length;
|
||||
ASSERT_GT(length, 1u);
|
||||
length -= 1u;
|
||||
}
|
||||
|
||||
static void makeOutputDimensionsUnspecified(Model* model) {
|
||||
for (auto i : model->outputIndexes) {
|
||||
auto& dims = model->operands[i].dimensions;
|
||||
std::fill(dims.begin(), dims.end(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
|
||||
const Request& request, MeasureTiming measure,
|
||||
sp<ExecutionCallback>& callback) {
|
||||
return preparedModel->execute_1_2(request, measure, callback);
|
||||
}
|
||||
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
|
||||
const Request& request, MeasureTiming measure,
|
||||
hidl_vec<OutputShape>* outputShapes,
|
||||
Timing* timing) {
|
||||
ErrorStatus result;
|
||||
Return<void> ret = preparedModel->executeSynchronously(
|
||||
request, measure,
|
||||
[&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
|
||||
const Timing& time) {
|
||||
result = error;
|
||||
*outputShapes = shapes;
|
||||
*timing = time;
|
||||
});
|
||||
if (!ret.isOk()) {
|
||||
return ErrorStatus::GENERAL_FAILURE;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
|
||||
const sp<IPreparedModel>& preparedModel) {
|
||||
return android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true);
|
||||
}
|
||||
enum class Executor { ASYNC, SYNC, BURST };
|
||||
|
||||
void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
|
||||
Executor executor, MeasureTiming measure, OutputType outputType) {
|
||||
// If output0 does not have size larger than one byte, we can not test with insufficient buffer.
|
||||
if (outputType == OutputType::INSUFFICIENT && !isOutputSizeGreaterThanOne(testModel, 0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Request request = createRequest(testModel);
|
||||
if (outputType == OutputType::INSUFFICIENT) {
|
||||
makeOutputInsufficientSize(/*outputIndex=*/0, &request);
|
||||
}
|
||||
|
||||
ErrorStatus executionStatus;
|
||||
hidl_vec<OutputShape> outputShapes;
|
||||
Timing timing;
|
||||
switch (executor) {
|
||||
case Executor::ASYNC: {
|
||||
SCOPED_TRACE("asynchronous");
|
||||
|
||||
// launch execution
|
||||
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
|
||||
Return<ErrorStatus> executionLaunchStatus =
|
||||
ExecutePreparedModel(preparedModel, request, measure, executionCallback);
|
||||
ASSERT_TRUE(executionLaunchStatus.isOk());
|
||||
EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
|
||||
|
||||
// retrieve execution status
|
||||
executionCallback->wait();
|
||||
executionStatus = executionCallback->getStatus();
|
||||
outputShapes = executionCallback->getOutputShapes();
|
||||
timing = executionCallback->getTiming();
|
||||
|
||||
break;
|
||||
}
|
||||
case Executor::SYNC: {
|
||||
SCOPED_TRACE("synchronous");
|
||||
|
||||
// execute
|
||||
Return<ErrorStatus> executionReturnStatus =
|
||||
ExecutePreparedModel(preparedModel, request, measure, &outputShapes, &timing);
|
||||
ASSERT_TRUE(executionReturnStatus.isOk());
|
||||
executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
|
||||
|
||||
break;
|
||||
}
|
||||
case Executor::BURST: {
|
||||
SCOPED_TRACE("burst");
|
||||
|
||||
// create burst
|
||||
const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
|
||||
CreateBurst(preparedModel);
|
||||
ASSERT_NE(nullptr, controller.get());
|
||||
|
||||
// create memory keys
|
||||
std::vector<intptr_t> keys(request.pools.size());
|
||||
for (size_t i = 0; i < keys.size(); ++i) {
|
||||
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
|
||||
}
|
||||
|
||||
// execute burst
|
||||
std::tie(executionStatus, outputShapes, timing) =
|
||||
controller->compute(request, measure, keys);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (outputType != OutputType::FULLY_SPECIFIED &&
|
||||
executionStatus == ErrorStatus::GENERAL_FAILURE) {
|
||||
LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
|
||||
"execute model that it does not support.";
|
||||
std::cout << "[ ] Early termination of test because vendor service cannot "
|
||||
"execute model that it does not support."
|
||||
<< std::endl;
|
||||
GTEST_SKIP();
|
||||
}
|
||||
if (measure == MeasureTiming::NO) {
|
||||
EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
|
||||
EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
|
||||
} else {
|
||||
if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
|
||||
EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
|
||||
}
|
||||
}
|
||||
|
||||
switch (outputType) {
|
||||
case OutputType::FULLY_SPECIFIED:
|
||||
// If the model output operands are fully specified, outputShapes must be either
|
||||
// either empty, or have the same number of elements as the number of outputs.
|
||||
ASSERT_EQ(ErrorStatus::NONE, executionStatus);
|
||||
ASSERT_TRUE(outputShapes.size() == 0 ||
|
||||
outputShapes.size() == testModel.outputIndexes.size());
|
||||
break;
|
||||
case OutputType::UNSPECIFIED:
|
||||
// If the model output operands are not fully specified, outputShapes must have
|
||||
// the same number of elements as the number of outputs.
|
||||
ASSERT_EQ(ErrorStatus::NONE, executionStatus);
|
||||
ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
|
||||
break;
|
||||
case OutputType::INSUFFICIENT:
|
||||
ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
|
||||
ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
|
||||
ASSERT_FALSE(outputShapes[0].isSufficient);
|
||||
return;
|
||||
}
|
||||
|
||||
// Go through all outputs, check returned output shapes.
|
||||
for (uint32_t i = 0; i < outputShapes.size(); i++) {
|
||||
EXPECT_TRUE(outputShapes[i].isSufficient);
|
||||
const auto& expect = testModel.operands[testModel.outputIndexes[i]].dimensions;
|
||||
const std::vector<uint32_t> actual = outputShapes[i].dimensions;
|
||||
EXPECT_EQ(expect, actual);
|
||||
}
|
||||
|
||||
// Retrieve execution results.
|
||||
const std::vector<TestBuffer> outputs = getOutputBuffers(request);
|
||||
|
||||
// We want "close-enough" results.
|
||||
checkResults(testModel, outputs);
|
||||
}
|
||||
|
||||
void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
|
||||
bool testDynamicOutputShape) {
|
||||
if (testDynamicOutputShape) {
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
|
||||
OutputType::UNSPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
|
||||
OutputType::INSUFFICIENT);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
|
||||
OutputType::INSUFFICIENT);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
|
||||
OutputType::INSUFFICIENT);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
|
||||
OutputType::INSUFFICIENT);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
|
||||
OutputType::INSUFFICIENT);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
|
||||
OutputType::INSUFFICIENT);
|
||||
} else {
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
|
||||
OutputType::FULLY_SPECIFIED);
|
||||
}
|
||||
}
|
||||
|
||||
void Execute(const sp<IDevice>& device, const TestModel& testModel, bool testDynamicOutputShape) {
|
||||
Model model = createModel(testModel);
|
||||
if (testDynamicOutputShape) {
|
||||
makeOutputDimensionsUnspecified(&model);
|
||||
}
|
||||
|
||||
sp<IPreparedModel> preparedModel;
|
||||
createPreparedModel(device, model, &preparedModel);
|
||||
if (preparedModel == nullptr) return;
|
||||
|
||||
EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape);
|
||||
}
|
||||
|
||||
void GeneratedTestBase::SetUp() {
|
||||
testing::TestWithParam<GeneratedTestParam>::SetUp();
|
||||
ASSERT_NE(kDevice, nullptr);
|
||||
}
|
||||
|
||||
std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
|
||||
return TestModelManager::get().getTestModels(filter);
|
||||
}
|
||||
|
||||
std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
|
||||
const auto& [namedDevice, namedModel] = info.param;
|
||||
return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
|
||||
}
|
||||
|
||||
// Tag for the generated tests
|
||||
class GeneratedTest : public GeneratedTestBase {};
|
||||
|
||||
// Tag for the dynamic output shape tests
|
||||
class DynamicOutputShapeTest : public GeneratedTest {};
|
||||
|
||||
TEST_P(GeneratedTest, Test) {
|
||||
Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false);
|
||||
}
|
||||
|
||||
TEST_P(DynamicOutputShapeTest, Test) {
|
||||
Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true);
|
||||
}
|
||||
|
||||
INSTANTIATE_GENERATED_TEST(GeneratedTest,
|
||||
[](const TestModel& testModel) { return !testModel.expectFailure; });
|
||||
|
||||
INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest,
|
||||
[](const TestModel& testModel) { return !testModel.expectFailure; });
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
65
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h
Normal file
65
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H
|
||||
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H
|
||||
|
||||
#include <android/hardware/neuralnetworks/1.2/IDevice.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/types.h>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
#include "1.0/Utils.h"
|
||||
#include "TestHarness.h"
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using NamedModel = Named<const test_helper::TestModel*>;
|
||||
using GeneratedTestParam = std::tuple<NamedDevice, NamedModel>;
|
||||
|
||||
class GeneratedTestBase : public testing::TestWithParam<GeneratedTestParam> {
|
||||
protected:
|
||||
void SetUp() override;
|
||||
const sp<IDevice> kDevice = getData(std::get<NamedDevice>(GetParam()));
|
||||
const test_helper::TestModel& kTestModel = *getData(std::get<NamedModel>(GetParam()));
|
||||
};
|
||||
|
||||
using FilterFn = std::function<bool(const test_helper::TestModel&)>;
|
||||
std::vector<NamedModel> getNamedModels(const FilterFn& filter);
|
||||
|
||||
std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info);
|
||||
|
||||
#define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \
|
||||
INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite, \
|
||||
testing::Combine(testing::ValuesIn(getNamedDevices()), \
|
||||
testing::ValuesIn(getNamedModels(filter))), \
|
||||
printGeneratedTest)
|
||||
|
||||
// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.
|
||||
// TODO: Clean up the hierarchy for ValidationTest.
|
||||
class ValidationTest : public GeneratedTestBase {};
|
||||
|
||||
Model createModel(const test_helper::TestModel& testModel);
|
||||
|
||||
void PrepareModel(const sp<IDevice>& device, const Model& model, sp<IPreparedModel>* preparedModel);
|
||||
|
||||
void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel,
|
||||
const test_helper::TestModel& testModel, bool testDynamicOutputShape);
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
|
||||
#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_GENERATED_TEST_HARNESS_H
|
||||
141
neuralnetworks/1.3/vts/functional/TestAssertions.cpp
Normal file
141
neuralnetworks/1.3/vts/functional/TestAssertions.cpp
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <android/hardware/neuralnetworks/1.2/types.h>
|
||||
#include "TestHarness.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2 {
|
||||
|
||||
// Make sure that the HIDL enums are compatible with the values defined in
|
||||
// frameworks/ml/nn/tools/test_generator/test_harness/include/TestHarness.h.
|
||||
using namespace test_helper;
|
||||
#define CHECK_TEST_ENUM(EnumType, enumValue) \
|
||||
static_assert(static_cast<EnumType>(Test##EnumType::enumValue) == EnumType::enumValue)
|
||||
|
||||
CHECK_TEST_ENUM(OperandType, FLOAT32);
|
||||
CHECK_TEST_ENUM(OperandType, INT32);
|
||||
CHECK_TEST_ENUM(OperandType, UINT32);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT32);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_INT32);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM);
|
||||
CHECK_TEST_ENUM(OperandType, BOOL);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_SYMM);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT16);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_BOOL8);
|
||||
CHECK_TEST_ENUM(OperandType, FLOAT16);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM_PER_CHANNEL);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_ASYMM);
|
||||
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM);
|
||||
|
||||
CHECK_TEST_ENUM(OperationType, ADD);
|
||||
CHECK_TEST_ENUM(OperationType, AVERAGE_POOL_2D);
|
||||
CHECK_TEST_ENUM(OperationType, CONCATENATION);
|
||||
CHECK_TEST_ENUM(OperationType, CONV_2D);
|
||||
CHECK_TEST_ENUM(OperationType, DEPTHWISE_CONV_2D);
|
||||
CHECK_TEST_ENUM(OperationType, DEPTH_TO_SPACE);
|
||||
CHECK_TEST_ENUM(OperationType, DEQUANTIZE);
|
||||
CHECK_TEST_ENUM(OperationType, EMBEDDING_LOOKUP);
|
||||
CHECK_TEST_ENUM(OperationType, FLOOR);
|
||||
CHECK_TEST_ENUM(OperationType, FULLY_CONNECTED);
|
||||
CHECK_TEST_ENUM(OperationType, HASHTABLE_LOOKUP);
|
||||
CHECK_TEST_ENUM(OperationType, L2_NORMALIZATION);
|
||||
CHECK_TEST_ENUM(OperationType, L2_POOL_2D);
|
||||
CHECK_TEST_ENUM(OperationType, LOCAL_RESPONSE_NORMALIZATION);
|
||||
CHECK_TEST_ENUM(OperationType, LOGISTIC);
|
||||
CHECK_TEST_ENUM(OperationType, LSH_PROJECTION);
|
||||
CHECK_TEST_ENUM(OperationType, LSTM);
|
||||
CHECK_TEST_ENUM(OperationType, MAX_POOL_2D);
|
||||
CHECK_TEST_ENUM(OperationType, MUL);
|
||||
CHECK_TEST_ENUM(OperationType, RELU);
|
||||
CHECK_TEST_ENUM(OperationType, RELU1);
|
||||
CHECK_TEST_ENUM(OperationType, RELU6);
|
||||
CHECK_TEST_ENUM(OperationType, RESHAPE);
|
||||
CHECK_TEST_ENUM(OperationType, RESIZE_BILINEAR);
|
||||
CHECK_TEST_ENUM(OperationType, RNN);
|
||||
CHECK_TEST_ENUM(OperationType, SOFTMAX);
|
||||
CHECK_TEST_ENUM(OperationType, SPACE_TO_DEPTH);
|
||||
CHECK_TEST_ENUM(OperationType, SVDF);
|
||||
CHECK_TEST_ENUM(OperationType, TANH);
|
||||
CHECK_TEST_ENUM(OperationType, BATCH_TO_SPACE_ND);
|
||||
CHECK_TEST_ENUM(OperationType, DIV);
|
||||
CHECK_TEST_ENUM(OperationType, MEAN);
|
||||
CHECK_TEST_ENUM(OperationType, PAD);
|
||||
CHECK_TEST_ENUM(OperationType, SPACE_TO_BATCH_ND);
|
||||
CHECK_TEST_ENUM(OperationType, SQUEEZE);
|
||||
CHECK_TEST_ENUM(OperationType, STRIDED_SLICE);
|
||||
CHECK_TEST_ENUM(OperationType, SUB);
|
||||
CHECK_TEST_ENUM(OperationType, TRANSPOSE);
|
||||
CHECK_TEST_ENUM(OperationType, ABS);
|
||||
CHECK_TEST_ENUM(OperationType, ARGMAX);
|
||||
CHECK_TEST_ENUM(OperationType, ARGMIN);
|
||||
CHECK_TEST_ENUM(OperationType, AXIS_ALIGNED_BBOX_TRANSFORM);
|
||||
CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_LSTM);
|
||||
CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_RNN);
|
||||
CHECK_TEST_ENUM(OperationType, BOX_WITH_NMS_LIMIT);
|
||||
CHECK_TEST_ENUM(OperationType, CAST);
|
||||
CHECK_TEST_ENUM(OperationType, CHANNEL_SHUFFLE);
|
||||
CHECK_TEST_ENUM(OperationType, DETECTION_POSTPROCESSING);
|
||||
CHECK_TEST_ENUM(OperationType, EQUAL);
|
||||
CHECK_TEST_ENUM(OperationType, EXP);
|
||||
CHECK_TEST_ENUM(OperationType, EXPAND_DIMS);
|
||||
CHECK_TEST_ENUM(OperationType, GATHER);
|
||||
CHECK_TEST_ENUM(OperationType, GENERATE_PROPOSALS);
|
||||
CHECK_TEST_ENUM(OperationType, GREATER);
|
||||
CHECK_TEST_ENUM(OperationType, GREATER_EQUAL);
|
||||
CHECK_TEST_ENUM(OperationType, GROUPED_CONV_2D);
|
||||
CHECK_TEST_ENUM(OperationType, HEATMAP_MAX_KEYPOINT);
|
||||
CHECK_TEST_ENUM(OperationType, INSTANCE_NORMALIZATION);
|
||||
CHECK_TEST_ENUM(OperationType, LESS);
|
||||
CHECK_TEST_ENUM(OperationType, LESS_EQUAL);
|
||||
CHECK_TEST_ENUM(OperationType, LOG);
|
||||
CHECK_TEST_ENUM(OperationType, LOGICAL_AND);
|
||||
CHECK_TEST_ENUM(OperationType, LOGICAL_NOT);
|
||||
CHECK_TEST_ENUM(OperationType, LOGICAL_OR);
|
||||
CHECK_TEST_ENUM(OperationType, LOG_SOFTMAX);
|
||||
CHECK_TEST_ENUM(OperationType, MAXIMUM);
|
||||
CHECK_TEST_ENUM(OperationType, MINIMUM);
|
||||
CHECK_TEST_ENUM(OperationType, NEG);
|
||||
CHECK_TEST_ENUM(OperationType, NOT_EQUAL);
|
||||
CHECK_TEST_ENUM(OperationType, PAD_V2);
|
||||
CHECK_TEST_ENUM(OperationType, POW);
|
||||
CHECK_TEST_ENUM(OperationType, PRELU);
|
||||
CHECK_TEST_ENUM(OperationType, QUANTIZE);
|
||||
CHECK_TEST_ENUM(OperationType, QUANTIZED_16BIT_LSTM);
|
||||
CHECK_TEST_ENUM(OperationType, RANDOM_MULTINOMIAL);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_ALL);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_ANY);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_MAX);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_MIN);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_PROD);
|
||||
CHECK_TEST_ENUM(OperationType, REDUCE_SUM);
|
||||
CHECK_TEST_ENUM(OperationType, ROI_ALIGN);
|
||||
CHECK_TEST_ENUM(OperationType, ROI_POOLING);
|
||||
CHECK_TEST_ENUM(OperationType, RSQRT);
|
||||
CHECK_TEST_ENUM(OperationType, SELECT);
|
||||
CHECK_TEST_ENUM(OperationType, SIN);
|
||||
CHECK_TEST_ENUM(OperationType, SLICE);
|
||||
CHECK_TEST_ENUM(OperationType, SPLIT);
|
||||
CHECK_TEST_ENUM(OperationType, SQRT);
|
||||
CHECK_TEST_ENUM(OperationType, TILE);
|
||||
CHECK_TEST_ENUM(OperationType, TOPK_V2);
|
||||
CHECK_TEST_ENUM(OperationType, TRANSPOSE_CONV_2D);
|
||||
CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_LSTM);
|
||||
CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_RNN);
|
||||
CHECK_TEST_ENUM(OperationType, RESIZE_NEAREST_NEIGHBOR);
|
||||
|
||||
#undef CHECK_TEST_ENUM
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2
|
||||
400
neuralnetworks/1.3/vts/functional/ValidateBurst.cpp
Normal file
400
neuralnetworks/1.3/vts/functional/ValidateBurst.cpp
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
/*
|
||||
* Copyright (C) 2019 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "neuralnetworks_hidl_hal_test"
|
||||
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
#include "1.2/Callbacks.h"
|
||||
#include "ExecutionBurstController.h"
|
||||
#include "ExecutionBurstServer.h"
|
||||
#include "GeneratedTestHarness.h"
|
||||
#include "TestHarness.h"
|
||||
#include "Utils.h"
|
||||
|
||||
#include <android-base/logging.h>
|
||||
#include <cstring>
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using nn::ExecutionBurstController;
|
||||
using nn::RequestChannelSender;
|
||||
using nn::ResultChannelReceiver;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::Request;
|
||||
using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback;
|
||||
|
||||
// This constant value represents the length of an FMQ that is large enough to
|
||||
// return a result from a burst execution for all of the generated test cases.
|
||||
constexpr size_t kExecutionBurstChannelLength = 1024;
|
||||
|
||||
// This constant value represents a length of an FMQ that is not large enough
|
||||
// to return a result from a burst execution for some of the generated test
|
||||
// cases.
|
||||
constexpr size_t kExecutionBurstChannelSmallLength = 8;
|
||||
|
||||
///////////////////////// UTILITY FUNCTIONS /////////////////////////
|
||||
|
||||
static bool badTiming(Timing timing) {
|
||||
return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
|
||||
}
|
||||
|
||||
static void createBurst(const sp<IPreparedModel>& preparedModel, const sp<IBurstCallback>& callback,
|
||||
std::unique_ptr<RequestChannelSender>* sender,
|
||||
std::unique_ptr<ResultChannelReceiver>* receiver,
|
||||
sp<IBurstContext>* context,
|
||||
size_t resultChannelLength = kExecutionBurstChannelLength) {
|
||||
ASSERT_NE(nullptr, preparedModel.get());
|
||||
ASSERT_NE(nullptr, sender);
|
||||
ASSERT_NE(nullptr, receiver);
|
||||
ASSERT_NE(nullptr, context);
|
||||
|
||||
// create FMQ objects
|
||||
auto [fmqRequestChannel, fmqRequestDescriptor] =
|
||||
RequestChannelSender::create(kExecutionBurstChannelLength, /*blocking=*/true);
|
||||
auto [fmqResultChannel, fmqResultDescriptor] =
|
||||
ResultChannelReceiver::create(resultChannelLength, /*blocking=*/true);
|
||||
ASSERT_NE(nullptr, fmqRequestChannel.get());
|
||||
ASSERT_NE(nullptr, fmqResultChannel.get());
|
||||
ASSERT_NE(nullptr, fmqRequestDescriptor);
|
||||
ASSERT_NE(nullptr, fmqResultDescriptor);
|
||||
|
||||
// configure burst
|
||||
ErrorStatus errorStatus;
|
||||
sp<IBurstContext> burstContext;
|
||||
const Return<void> ret = preparedModel->configureExecutionBurst(
|
||||
callback, *fmqRequestDescriptor, *fmqResultDescriptor,
|
||||
[&errorStatus, &burstContext](ErrorStatus status, const sp<IBurstContext>& context) {
|
||||
errorStatus = status;
|
||||
burstContext = context;
|
||||
});
|
||||
ASSERT_TRUE(ret.isOk());
|
||||
ASSERT_EQ(ErrorStatus::NONE, errorStatus);
|
||||
ASSERT_NE(nullptr, burstContext.get());
|
||||
|
||||
// return values
|
||||
*sender = std::move(fmqRequestChannel);
|
||||
*receiver = std::move(fmqResultChannel);
|
||||
*context = burstContext;
|
||||
}
|
||||
|
||||
static void createBurstWithResultChannelLength(
|
||||
const sp<IPreparedModel>& preparedModel, size_t resultChannelLength,
|
||||
std::shared_ptr<ExecutionBurstController>* controller) {
|
||||
ASSERT_NE(nullptr, preparedModel.get());
|
||||
ASSERT_NE(nullptr, controller);
|
||||
|
||||
// create FMQ objects
|
||||
std::unique_ptr<RequestChannelSender> sender;
|
||||
std::unique_ptr<ResultChannelReceiver> receiver;
|
||||
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
|
||||
sp<IBurstContext> context;
|
||||
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context,
|
||||
resultChannelLength));
|
||||
ASSERT_NE(nullptr, sender.get());
|
||||
ASSERT_NE(nullptr, receiver.get());
|
||||
ASSERT_NE(nullptr, context.get());
|
||||
|
||||
// return values
|
||||
*controller = std::make_shared<ExecutionBurstController>(std::move(sender), std::move(receiver),
|
||||
context, callback);
|
||||
}
|
||||
|
||||
// Primary validation function. This function will take a valid serialized
|
||||
// request, apply a mutation to it to invalidate the serialized request, then
|
||||
// pass it to interface calls that use the serialized request. Note that the
|
||||
// serialized request here is passed by value, and any mutation to the
|
||||
// serialized request does not leave this function.
|
||||
static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver,
|
||||
const std::string& message, std::vector<FmqRequestDatum> serialized,
|
||||
const std::function<void(std::vector<FmqRequestDatum>*)>& mutation) {
|
||||
mutation(&serialized);
|
||||
|
||||
// skip if packet is too large to send
|
||||
if (serialized.size() > kExecutionBurstChannelLength) {
|
||||
return;
|
||||
}
|
||||
|
||||
SCOPED_TRACE(message);
|
||||
|
||||
// send invalid packet
|
||||
ASSERT_TRUE(sender->sendPacket(serialized));
|
||||
|
||||
// receive error
|
||||
auto results = receiver->getBlocking();
|
||||
ASSERT_TRUE(results.has_value());
|
||||
const auto [status, outputShapes, timing] = std::move(*results);
|
||||
EXPECT_NE(ErrorStatus::NONE, status);
|
||||
EXPECT_EQ(0u, outputShapes.size());
|
||||
EXPECT_TRUE(badTiming(timing));
|
||||
}
|
||||
|
||||
// For validation, valid packet entries are mutated to invalid packet entries,
|
||||
// or invalid packet entries are inserted into valid packets. This function
|
||||
// creates pre-set invalid packet entries for convenience.
|
||||
static std::vector<FmqRequestDatum> createBadRequestPacketEntries() {
|
||||
const FmqRequestDatum::PacketInformation packetInformation = {
|
||||
/*.packetSize=*/10, /*.numberOfInputOperands=*/10, /*.numberOfOutputOperands=*/10,
|
||||
/*.numberOfPools=*/10};
|
||||
const FmqRequestDatum::OperandInformation operandInformation = {
|
||||
/*.hasNoValue=*/false, /*.location=*/{}, /*.numberOfDimensions=*/10};
|
||||
const int32_t invalidPoolIdentifier = std::numeric_limits<int32_t>::max();
|
||||
std::vector<FmqRequestDatum> bad(7);
|
||||
bad[0].packetInformation(packetInformation);
|
||||
bad[1].inputOperandInformation(operandInformation);
|
||||
bad[2].inputOperandDimensionValue(0);
|
||||
bad[3].outputOperandInformation(operandInformation);
|
||||
bad[4].outputOperandDimensionValue(0);
|
||||
bad[5].poolIdentifier(invalidPoolIdentifier);
|
||||
bad[6].measureTiming(MeasureTiming::YES);
|
||||
return bad;
|
||||
}
|
||||
|
||||
// For validation, valid packet entries are mutated to invalid packet entries,
|
||||
// or invalid packet entries are inserted into valid packets. This function
|
||||
// retrieves pre-set invalid packet entries for convenience. This function
|
||||
// caches these data so they can be reused on subsequent validation checks.
|
||||
static const std::vector<FmqRequestDatum>& getBadRequestPacketEntries() {
|
||||
static const std::vector<FmqRequestDatum> bad = createBadRequestPacketEntries();
|
||||
return bad;
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE DATUM ////////////////////////////////////
|
||||
|
||||
static void removeDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
|
||||
const std::vector<FmqRequestDatum>& serialized) {
|
||||
for (size_t index = 0; index < serialized.size(); ++index) {
|
||||
const std::string message = "removeDatum: removed datum at index " + std::to_string(index);
|
||||
validate(sender, receiver, message, serialized,
|
||||
[index](std::vector<FmqRequestDatum>* serialized) {
|
||||
serialized->erase(serialized->begin() + index);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// ADD DATUM ////////////////////////////////////
|
||||
|
||||
static void addDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
|
||||
const std::vector<FmqRequestDatum>& serialized) {
|
||||
const std::vector<FmqRequestDatum>& extra = getBadRequestPacketEntries();
|
||||
for (size_t index = 0; index <= serialized.size(); ++index) {
|
||||
for (size_t type = 0; type < extra.size(); ++type) {
|
||||
const std::string message = "addDatum: added datum type " + std::to_string(type) +
|
||||
" at index " + std::to_string(index);
|
||||
validate(sender, receiver, message, serialized,
|
||||
[index, type, &extra](std::vector<FmqRequestDatum>* serialized) {
|
||||
serialized->insert(serialized->begin() + index, extra[type]);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// MUTATE DATUM ////////////////////////////////////
|
||||
|
||||
static bool interestingCase(const FmqRequestDatum& lhs, const FmqRequestDatum& rhs) {
|
||||
using Discriminator = FmqRequestDatum::hidl_discriminator;
|
||||
|
||||
const bool differentValues = (lhs != rhs);
|
||||
const bool sameDiscriminator = (lhs.getDiscriminator() == rhs.getDiscriminator());
|
||||
const auto discriminator = rhs.getDiscriminator();
|
||||
const bool isDimensionValue = (discriminator == Discriminator::inputOperandDimensionValue ||
|
||||
discriminator == Discriminator::outputOperandDimensionValue);
|
||||
|
||||
return differentValues && !(sameDiscriminator && isDimensionValue);
|
||||
}
|
||||
|
||||
static void mutateDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
|
||||
const std::vector<FmqRequestDatum>& serialized) {
|
||||
const std::vector<FmqRequestDatum>& change = getBadRequestPacketEntries();
|
||||
for (size_t index = 0; index < serialized.size(); ++index) {
|
||||
for (size_t type = 0; type < change.size(); ++type) {
|
||||
if (interestingCase(serialized[index], change[type])) {
|
||||
const std::string message = "mutateDatum: changed datum at index " +
|
||||
std::to_string(index) + " to datum type " +
|
||||
std::to_string(type);
|
||||
validate(sender, receiver, message, serialized,
|
||||
[index, type, &change](std::vector<FmqRequestDatum>* serialized) {
|
||||
(*serialized)[index] = change[type];
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// BURST VALIATION TESTS ////////////////////////////////////
|
||||
|
||||
static void validateBurstSerialization(const sp<IPreparedModel>& preparedModel,
|
||||
const Request& request) {
|
||||
// create burst
|
||||
std::unique_ptr<RequestChannelSender> sender;
|
||||
std::unique_ptr<ResultChannelReceiver> receiver;
|
||||
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
|
||||
sp<IBurstContext> context;
|
||||
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
|
||||
ASSERT_NE(nullptr, sender.get());
|
||||
ASSERT_NE(nullptr, receiver.get());
|
||||
ASSERT_NE(nullptr, context.get());
|
||||
|
||||
// load memory into callback slots
|
||||
std::vector<intptr_t> keys;
|
||||
keys.reserve(request.pools.size());
|
||||
std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
|
||||
[](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
|
||||
const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
|
||||
|
||||
// ensure slot std::numeric_limits<int32_t>::max() doesn't exist (for
|
||||
// subsequent slot validation testing)
|
||||
ASSERT_TRUE(std::all_of(slots.begin(), slots.end(), [](int32_t slot) {
|
||||
return slot != std::numeric_limits<int32_t>::max();
|
||||
}));
|
||||
|
||||
// serialize the request
|
||||
const auto serialized = android::nn::serialize(request, MeasureTiming::YES, slots);
|
||||
|
||||
// validations
|
||||
removeDatumTest(sender.get(), receiver.get(), serialized);
|
||||
addDatumTest(sender.get(), receiver.get(), serialized);
|
||||
mutateDatumTest(sender.get(), receiver.get(), serialized);
|
||||
}
|
||||
|
||||
// This test validates that when the Result message size exceeds length of the
|
||||
// result FMQ, the service instance gracefully fails and returns an error.
|
||||
static void validateBurstFmqLength(const sp<IPreparedModel>& preparedModel,
|
||||
const Request& request) {
|
||||
// create regular burst
|
||||
std::shared_ptr<ExecutionBurstController> controllerRegular;
|
||||
ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
|
||||
preparedModel, kExecutionBurstChannelLength, &controllerRegular));
|
||||
ASSERT_NE(nullptr, controllerRegular.get());
|
||||
|
||||
// create burst with small output channel
|
||||
std::shared_ptr<ExecutionBurstController> controllerSmall;
|
||||
ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
|
||||
preparedModel, kExecutionBurstChannelSmallLength, &controllerSmall));
|
||||
ASSERT_NE(nullptr, controllerSmall.get());
|
||||
|
||||
// load memory into callback slots
|
||||
std::vector<intptr_t> keys(request.pools.size());
|
||||
for (size_t i = 0; i < keys.size(); ++i) {
|
||||
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
|
||||
}
|
||||
|
||||
// collect serialized result by running regular burst
|
||||
const auto [statusRegular, outputShapesRegular, timingRegular] =
|
||||
controllerRegular->compute(request, MeasureTiming::NO, keys);
|
||||
|
||||
// skip test if regular burst output isn't useful for testing a failure
|
||||
// caused by having too small of a length for the result FMQ
|
||||
const std::vector<FmqResultDatum> serialized =
|
||||
android::nn::serialize(statusRegular, outputShapesRegular, timingRegular);
|
||||
if (statusRegular != ErrorStatus::NONE ||
|
||||
serialized.size() <= kExecutionBurstChannelSmallLength) {
|
||||
return;
|
||||
}
|
||||
|
||||
// by this point, execution should fail because the result channel isn't
|
||||
// large enough to return the serialized result
|
||||
const auto [statusSmall, outputShapesSmall, timingSmall] =
|
||||
controllerSmall->compute(request, MeasureTiming::NO, keys);
|
||||
EXPECT_NE(ErrorStatus::NONE, statusSmall);
|
||||
EXPECT_EQ(0u, outputShapesSmall.size());
|
||||
EXPECT_TRUE(badTiming(timingSmall));
|
||||
}
|
||||
|
||||
static bool isSanitized(const FmqResultDatum& datum) {
|
||||
using Discriminator = FmqResultDatum::hidl_discriminator;
|
||||
|
||||
// check to ensure the padding values in the returned
|
||||
// FmqResultDatum::OperandInformation are initialized to 0
|
||||
if (datum.getDiscriminator() == Discriminator::operandInformation) {
|
||||
static_assert(
|
||||
offsetof(FmqResultDatum::OperandInformation, isSufficient) == 0,
|
||||
"unexpected value for offset of FmqResultDatum::OperandInformation::isSufficient");
|
||||
static_assert(
|
||||
sizeof(FmqResultDatum::OperandInformation::isSufficient) == 1,
|
||||
"unexpected value for size of FmqResultDatum::OperandInformation::isSufficient");
|
||||
static_assert(offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) == 4,
|
||||
"unexpected value for offset of "
|
||||
"FmqResultDatum::OperandInformation::numberOfDimensions");
|
||||
static_assert(sizeof(FmqResultDatum::OperandInformation::numberOfDimensions) == 4,
|
||||
"unexpected value for size of "
|
||||
"FmqResultDatum::OperandInformation::numberOfDimensions");
|
||||
static_assert(sizeof(FmqResultDatum::OperandInformation) == 8,
|
||||
"unexpected value for size of "
|
||||
"FmqResultDatum::OperandInformation");
|
||||
|
||||
constexpr size_t paddingOffset =
|
||||
offsetof(FmqResultDatum::OperandInformation, isSufficient) +
|
||||
sizeof(FmqResultDatum::OperandInformation::isSufficient);
|
||||
constexpr size_t paddingSize =
|
||||
offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) - paddingOffset;
|
||||
|
||||
FmqResultDatum::OperandInformation initialized{};
|
||||
std::memset(&initialized, 0, sizeof(initialized));
|
||||
|
||||
const char* initializedPaddingStart =
|
||||
reinterpret_cast<const char*>(&initialized) + paddingOffset;
|
||||
const char* datumPaddingStart =
|
||||
reinterpret_cast<const char*>(&datum.operandInformation()) + paddingOffset;
|
||||
|
||||
return std::memcmp(datumPaddingStart, initializedPaddingStart, paddingSize) == 0;
|
||||
}
|
||||
|
||||
// there are no other padding initialization checks required, so return true
|
||||
// for any sum-type that isn't FmqResultDatum::OperandInformation
|
||||
return true;
|
||||
}
|
||||
|
||||
static void validateBurstSanitized(const sp<IPreparedModel>& preparedModel,
|
||||
const Request& request) {
|
||||
// create burst
|
||||
std::unique_ptr<RequestChannelSender> sender;
|
||||
std::unique_ptr<ResultChannelReceiver> receiver;
|
||||
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
|
||||
sp<IBurstContext> context;
|
||||
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
|
||||
ASSERT_NE(nullptr, sender.get());
|
||||
ASSERT_NE(nullptr, receiver.get());
|
||||
ASSERT_NE(nullptr, context.get());
|
||||
|
||||
// load memory into callback slots
|
||||
std::vector<intptr_t> keys;
|
||||
keys.reserve(request.pools.size());
|
||||
std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
|
||||
[](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
|
||||
const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
|
||||
|
||||
// send valid request
|
||||
ASSERT_TRUE(sender->send(request, MeasureTiming::YES, slots));
|
||||
|
||||
// receive valid result
|
||||
auto serialized = receiver->getPacketBlocking();
|
||||
ASSERT_TRUE(serialized.has_value());
|
||||
|
||||
// sanitize result
|
||||
ASSERT_TRUE(std::all_of(serialized->begin(), serialized->end(), isSanitized))
|
||||
<< "The result serialized data is not properly sanitized";
|
||||
}
|
||||
|
||||
///////////////////////////// ENTRY POINT //////////////////////////////////
|
||||
|
||||
void validateBurst(const sp<IPreparedModel>& preparedModel, const Request& request) {
|
||||
ASSERT_NO_FATAL_FAILURE(validateBurstSerialization(preparedModel, request));
|
||||
ASSERT_NO_FATAL_FAILURE(validateBurstFmqLength(preparedModel, request));
|
||||
ASSERT_NO_FATAL_FAILURE(validateBurstSanitized(preparedModel, request));
|
||||
}
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
713
neuralnetworks/1.3/vts/functional/ValidateModel.cpp
Normal file
713
neuralnetworks/1.3/vts/functional/ValidateModel.cpp
Normal file
|
|
@ -0,0 +1,713 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "neuralnetworks_hidl_hal_test"
|
||||
|
||||
#include "1.0/Utils.h"
|
||||
#include "1.2/Callbacks.h"
|
||||
#include "GeneratedTestHarness.h"
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using implementation::PreparedModelCallback;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::OperandLifeTime;
|
||||
using V1_1::ExecutionPreference;
|
||||
using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
|
||||
|
||||
///////////////////////// UTILITY FUNCTIONS /////////////////////////
|
||||
|
||||
static void validateGetSupportedOperations(const sp<IDevice>& device, const std::string& message,
|
||||
const Model& model) {
|
||||
SCOPED_TRACE(message + " [getSupportedOperations_1_2]");
|
||||
|
||||
Return<void> ret = device->getSupportedOperations_1_2(
|
||||
model, [&](ErrorStatus status, const hidl_vec<bool>&) {
|
||||
EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
|
||||
});
|
||||
EXPECT_TRUE(ret.isOk());
|
||||
}
|
||||
|
||||
static void validatePrepareModel(const sp<IDevice>& device, const std::string& message,
|
||||
const Model& model, ExecutionPreference preference) {
|
||||
SCOPED_TRACE(message + " [prepareModel_1_2]");
|
||||
|
||||
sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
|
||||
Return<ErrorStatus> prepareLaunchStatus =
|
||||
device->prepareModel_1_2(model, preference, hidl_vec<hidl_handle>(),
|
||||
hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
|
||||
ASSERT_TRUE(prepareLaunchStatus.isOk());
|
||||
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
|
||||
|
||||
preparedModelCallback->wait();
|
||||
ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
|
||||
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
|
||||
sp<IPreparedModel> preparedModel = getPreparedModel_1_2(preparedModelCallback);
|
||||
ASSERT_EQ(nullptr, preparedModel.get());
|
||||
}
|
||||
|
||||
static bool validExecutionPreference(ExecutionPreference preference) {
|
||||
return preference == ExecutionPreference::LOW_POWER ||
|
||||
preference == ExecutionPreference::FAST_SINGLE_ANSWER ||
|
||||
preference == ExecutionPreference::SUSTAINED_SPEED;
|
||||
}
|
||||
|
||||
// Primary validation function. This function will take a valid model, apply a
|
||||
// mutation to it to invalidate the model, then pass it to interface calls that
|
||||
// use the model. Note that the model here is passed by value, and any mutation
|
||||
// to the model does not leave this function.
|
||||
static void validate(const sp<IDevice>& device, const std::string& message, Model model,
|
||||
const std::function<void(Model*)>& mutation,
|
||||
ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) {
|
||||
mutation(&model);
|
||||
if (validExecutionPreference(preference)) {
|
||||
validateGetSupportedOperations(device, message, model);
|
||||
}
|
||||
validatePrepareModel(device, message, model, preference);
|
||||
}
|
||||
|
||||
static uint32_t addOperand(Model* model) {
|
||||
return hidl_vec_push_back(&model->operands,
|
||||
{
|
||||
.type = OperandType::INT32,
|
||||
.dimensions = {},
|
||||
.numberOfConsumers = 0,
|
||||
.scale = 0.0f,
|
||||
.zeroPoint = 0,
|
||||
.lifetime = OperandLifeTime::MODEL_INPUT,
|
||||
.location = {.poolIndex = 0, .offset = 0, .length = 0},
|
||||
});
|
||||
}
|
||||
|
||||
static uint32_t addOperand(Model* model, OperandLifeTime lifetime) {
|
||||
uint32_t index = addOperand(model);
|
||||
model->operands[index].numberOfConsumers = 1;
|
||||
model->operands[index].lifetime = lifetime;
|
||||
return index;
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE MODEL OPERAND TYPE /////////////////////////
|
||||
|
||||
static const uint32_t invalidOperandTypes[] = {
|
||||
static_cast<uint32_t>(OperandTypeRange::FUNDAMENTAL_MIN) - 1,
|
||||
static_cast<uint32_t>(OperandTypeRange::FUNDAMENTAL_MAX) + 1,
|
||||
static_cast<uint32_t>(OperandTypeRange::OEM_MIN) - 1,
|
||||
static_cast<uint32_t>(OperandTypeRange::OEM_MAX) + 1,
|
||||
};
|
||||
|
||||
static void mutateOperandTypeTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
for (uint32_t invalidOperandType : invalidOperandTypes) {
|
||||
const std::string message = "mutateOperandTypeTest: operand " +
|
||||
std::to_string(operand) + " set to value " +
|
||||
std::to_string(invalidOperandType);
|
||||
validate(device, message, model, [operand, invalidOperandType](Model* model) {
|
||||
model->operands[operand].type = static_cast<OperandType>(invalidOperandType);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE OPERAND RANK /////////////////////////
|
||||
|
||||
static uint32_t getInvalidRank(OperandType type) {
|
||||
switch (type) {
|
||||
case OperandType::FLOAT16:
|
||||
case OperandType::FLOAT32:
|
||||
case OperandType::INT32:
|
||||
case OperandType::UINT32:
|
||||
case OperandType::BOOL:
|
||||
return 1;
|
||||
case OperandType::TENSOR_BOOL8:
|
||||
case OperandType::TENSOR_FLOAT16:
|
||||
case OperandType::TENSOR_FLOAT32:
|
||||
case OperandType::TENSOR_INT32:
|
||||
case OperandType::TENSOR_QUANT8_ASYMM:
|
||||
case OperandType::TENSOR_QUANT8_SYMM:
|
||||
case OperandType::TENSOR_QUANT16_ASYMM:
|
||||
case OperandType::TENSOR_QUANT16_SYMM:
|
||||
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
|
||||
return 0;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void mutateOperandRankTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
const uint32_t invalidRank = getInvalidRank(model.operands[operand].type);
|
||||
if (invalidRank == 0) {
|
||||
continue;
|
||||
}
|
||||
const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) +
|
||||
" has rank of " + std::to_string(invalidRank);
|
||||
validate(device, message, model, [operand, invalidRank](Model* model) {
|
||||
model->operands[operand].dimensions = std::vector<uint32_t>(invalidRank, 0);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE OPERAND SCALE /////////////////////////
|
||||
|
||||
static float getInvalidScale(OperandType type) {
|
||||
switch (type) {
|
||||
case OperandType::FLOAT16:
|
||||
case OperandType::FLOAT32:
|
||||
case OperandType::INT32:
|
||||
case OperandType::UINT32:
|
||||
case OperandType::BOOL:
|
||||
case OperandType::TENSOR_BOOL8:
|
||||
case OperandType::TENSOR_FLOAT16:
|
||||
case OperandType::TENSOR_FLOAT32:
|
||||
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
|
||||
return 1.0f;
|
||||
case OperandType::TENSOR_INT32:
|
||||
return -1.0f;
|
||||
case OperandType::TENSOR_QUANT8_SYMM:
|
||||
case OperandType::TENSOR_QUANT8_ASYMM:
|
||||
case OperandType::TENSOR_QUANT16_ASYMM:
|
||||
case OperandType::TENSOR_QUANT16_SYMM:
|
||||
return 0.0f;
|
||||
default:
|
||||
return 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
static void mutateOperandScaleTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
const float invalidScale = getInvalidScale(model.operands[operand].type);
|
||||
const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) +
|
||||
" has scale of " + std::to_string(invalidScale);
|
||||
validate(device, message, model, [operand, invalidScale](Model* model) {
|
||||
model->operands[operand].scale = invalidScale;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE OPERAND ZERO POINT /////////////////////////
|
||||
|
||||
static std::vector<int32_t> getInvalidZeroPoints(OperandType type) {
|
||||
switch (type) {
|
||||
case OperandType::FLOAT16:
|
||||
case OperandType::FLOAT32:
|
||||
case OperandType::INT32:
|
||||
case OperandType::UINT32:
|
||||
case OperandType::BOOL:
|
||||
case OperandType::TENSOR_BOOL8:
|
||||
case OperandType::TENSOR_FLOAT16:
|
||||
case OperandType::TENSOR_FLOAT32:
|
||||
case OperandType::TENSOR_INT32:
|
||||
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
|
||||
return {1};
|
||||
case OperandType::TENSOR_QUANT8_ASYMM:
|
||||
return {-1, 256};
|
||||
case OperandType::TENSOR_QUANT8_SYMM:
|
||||
return {-129, -1, 1, 128};
|
||||
case OperandType::TENSOR_QUANT16_ASYMM:
|
||||
return {-1, 65536};
|
||||
case OperandType::TENSOR_QUANT16_SYMM:
|
||||
return {-32769, -1, 1, 32768};
|
||||
default:
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
static void mutateOperandZeroPointTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
const std::vector<int32_t> invalidZeroPoints =
|
||||
getInvalidZeroPoints(model.operands[operand].type);
|
||||
for (int32_t invalidZeroPoint : invalidZeroPoints) {
|
||||
const std::string message = "mutateOperandZeroPointTest: operand " +
|
||||
std::to_string(operand) + " has zero point of " +
|
||||
std::to_string(invalidZeroPoint);
|
||||
validate(device, message, model, [operand, invalidZeroPoint](Model* model) {
|
||||
model->operands[operand].zeroPoint = invalidZeroPoint;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE EXTRA ??? /////////////////////////
|
||||
|
||||
// TODO: Operand::lifetime
|
||||
// TODO: Operand::location
|
||||
|
||||
///////////////////////// VALIDATE OPERATION OPERAND TYPE /////////////////////////
|
||||
|
||||
static void mutateOperand(Operand* operand, OperandType type) {
|
||||
Operand newOperand = *operand;
|
||||
newOperand.type = type;
|
||||
switch (type) {
|
||||
case OperandType::FLOAT16:
|
||||
case OperandType::FLOAT32:
|
||||
case OperandType::INT32:
|
||||
case OperandType::UINT32:
|
||||
case OperandType::BOOL:
|
||||
newOperand.dimensions = hidl_vec<uint32_t>();
|
||||
newOperand.scale = 0.0f;
|
||||
newOperand.zeroPoint = 0;
|
||||
break;
|
||||
case OperandType::TENSOR_BOOL8:
|
||||
case OperandType::TENSOR_FLOAT16:
|
||||
case OperandType::TENSOR_FLOAT32:
|
||||
newOperand.dimensions =
|
||||
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
|
||||
newOperand.scale = 0.0f;
|
||||
newOperand.zeroPoint = 0;
|
||||
break;
|
||||
case OperandType::TENSOR_INT32:
|
||||
newOperand.dimensions =
|
||||
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
|
||||
newOperand.zeroPoint = 0;
|
||||
break;
|
||||
case OperandType::TENSOR_QUANT8_ASYMM:
|
||||
case OperandType::TENSOR_QUANT8_SYMM:
|
||||
case OperandType::TENSOR_QUANT16_ASYMM:
|
||||
case OperandType::TENSOR_QUANT16_SYMM:
|
||||
newOperand.dimensions =
|
||||
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
|
||||
newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f;
|
||||
break;
|
||||
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: {
|
||||
newOperand.dimensions =
|
||||
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
|
||||
newOperand.scale = 0.0f;
|
||||
newOperand.zeroPoint = 0;
|
||||
|
||||
SymmPerChannelQuantParams channelQuant;
|
||||
channelQuant.channelDim = 0;
|
||||
channelQuant.scales = hidl_vec<float>(
|
||||
operand->dimensions.size() > 0 ? static_cast<size_t>(operand->dimensions[0])
|
||||
: 0);
|
||||
for (size_t i = 0; i < channelQuant.scales.size(); ++i) {
|
||||
channelQuant.scales[i] = 1.0f;
|
||||
}
|
||||
newOperand.extraParams.channelQuant(std::move(channelQuant));
|
||||
} break;
|
||||
case OperandType::OEM:
|
||||
case OperandType::TENSOR_OEM_BYTE:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
*operand = newOperand;
|
||||
}
|
||||
|
||||
static bool mutateOperationOperandTypeSkip(size_t operand, OperandType type, const Model& model) {
|
||||
// Do not test OEM types
|
||||
if (type == model.operands[operand].type || type == OperandType::OEM ||
|
||||
type == OperandType::TENSOR_OEM_BYTE) {
|
||||
return true;
|
||||
}
|
||||
for (const Operation& operation : model.operations) {
|
||||
// Skip mutateOperationOperandTypeTest for the following operations.
|
||||
// - LSH_PROJECTION's second argument is allowed to have any type.
|
||||
// - ARGMIN and ARGMAX's first argument can be any of
|
||||
// TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM).
|
||||
// - CAST's argument can be any of TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM).
|
||||
// - RANDOM_MULTINOMIAL's argument can be either TENSOR_FLOAT16 or TENSOR_FLOAT32.
|
||||
// - DEQUANTIZE input can be any of
|
||||
// TENSOR_(QUANT8_ASYMM|QUANT8_SYMM|QUANT8_SYMM_PER_CHANNEL), output can
|
||||
// be of either TENSOR_FLOAT16 or TENSOR_FLOAT32.
|
||||
// - QUANTIZE input can be either TENSOR_FLOAT16 or TENSOR_FLOAT32
|
||||
// - CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
|
||||
// - DEPTHWISE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
|
||||
// - GROUPED_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
|
||||
// - TRANSPOSE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
|
||||
switch (operation.type) {
|
||||
case OperationType::LSH_PROJECTION: {
|
||||
if (operand == operation.inputs[1]) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::CAST:
|
||||
case OperationType::ARGMAX:
|
||||
case OperationType::ARGMIN: {
|
||||
if (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32 ||
|
||||
type == OperandType::TENSOR_INT32 || type == OperandType::TENSOR_QUANT8_ASYMM) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::QUANTIZE:
|
||||
case OperationType::RANDOM_MULTINOMIAL: {
|
||||
if (operand == operation.inputs[0] &&
|
||||
(type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::DEQUANTIZE: {
|
||||
if (operand == operation.inputs[0] &&
|
||||
(type == OperandType::TENSOR_QUANT8_ASYMM ||
|
||||
type == OperandType::TENSOR_QUANT8_SYMM ||
|
||||
type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) {
|
||||
return true;
|
||||
}
|
||||
if (operand == operation.outputs[0] &&
|
||||
(type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::TRANSPOSE_CONV_2D:
|
||||
case OperationType::GROUPED_CONV_2D:
|
||||
case OperationType::DEPTHWISE_CONV_2D:
|
||||
case OperationType::CONV_2D: {
|
||||
if (operand == operation.inputs[1] &&
|
||||
(type == OperandType::TENSOR_QUANT8_ASYMM ||
|
||||
type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void mutateOperationOperandTypeTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
for (OperandType invalidOperandType : hidl_enum_range<OperandType>{}) {
|
||||
if (mutateOperationOperandTypeSkip(operand, invalidOperandType, model)) {
|
||||
continue;
|
||||
}
|
||||
const std::string message = "mutateOperationOperandTypeTest: operand " +
|
||||
std::to_string(operand) + " set to type " +
|
||||
toString(invalidOperandType);
|
||||
validate(device, message, model, [operand, invalidOperandType](Model* model) {
|
||||
mutateOperand(&model->operands[operand], invalidOperandType);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE MODEL OPERATION TYPE /////////////////////////
|
||||
|
||||
static const uint32_t invalidOperationTypes[] = {
|
||||
static_cast<uint32_t>(OperationTypeRange::FUNDAMENTAL_MAX) + 1,
|
||||
static_cast<uint32_t>(OperationTypeRange::OEM_MIN) - 1,
|
||||
static_cast<uint32_t>(OperationTypeRange::OEM_MAX) + 1,
|
||||
};
|
||||
|
||||
static void mutateOperationTypeTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
for (uint32_t invalidOperationType : invalidOperationTypes) {
|
||||
const std::string message = "mutateOperationTypeTest: operation " +
|
||||
std::to_string(operation) + " set to value " +
|
||||
std::to_string(invalidOperationType);
|
||||
validate(device, message, model, [operation, invalidOperationType](Model* model) {
|
||||
model->operations[operation].type =
|
||||
static_cast<OperationType>(invalidOperationType);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE MODEL OPERATION INPUT OPERAND INDEX /////////////////////////
|
||||
|
||||
static void mutateOperationInputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
const uint32_t invalidOperand = model.operands.size();
|
||||
for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
|
||||
const std::string message = "mutateOperationInputOperandIndexTest: operation " +
|
||||
std::to_string(operation) + " input " +
|
||||
std::to_string(input);
|
||||
validate(device, message, model, [operation, input, invalidOperand](Model* model) {
|
||||
model->operations[operation].inputs[input] = invalidOperand;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE MODEL OPERATION OUTPUT OPERAND INDEX /////////////////////////
|
||||
|
||||
static void mutateOperationOutputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
const uint32_t invalidOperand = model.operands.size();
|
||||
for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
|
||||
const std::string message = "mutateOperationOutputOperandIndexTest: operation " +
|
||||
std::to_string(operation) + " output " +
|
||||
std::to_string(output);
|
||||
validate(device, message, model, [operation, output, invalidOperand](Model* model) {
|
||||
model->operations[operation].outputs[output] = invalidOperand;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE OPERAND FROM EVERYTHING /////////////////////////
|
||||
|
||||
static void removeValueAndDecrementGreaterValues(hidl_vec<uint32_t>* vec, uint32_t value) {
|
||||
if (vec) {
|
||||
// remove elements matching "value"
|
||||
auto last = std::remove(vec->begin(), vec->end(), value);
|
||||
vec->resize(std::distance(vec->begin(), last));
|
||||
|
||||
// decrement elements exceeding "value"
|
||||
std::transform(vec->begin(), vec->end(), vec->begin(),
|
||||
[value](uint32_t v) { return v > value ? v-- : v; });
|
||||
}
|
||||
}
|
||||
|
||||
static void removeOperand(Model* model, uint32_t index) {
|
||||
hidl_vec_removeAt(&model->operands, index);
|
||||
for (Operation& operation : model->operations) {
|
||||
removeValueAndDecrementGreaterValues(&operation.inputs, index);
|
||||
removeValueAndDecrementGreaterValues(&operation.outputs, index);
|
||||
}
|
||||
removeValueAndDecrementGreaterValues(&model->inputIndexes, index);
|
||||
removeValueAndDecrementGreaterValues(&model->outputIndexes, index);
|
||||
}
|
||||
|
||||
static bool removeOperandSkip(size_t operand, const Model& model) {
|
||||
for (const Operation& operation : model.operations) {
|
||||
// Skip removeOperandTest for the following operations.
|
||||
// - SPLIT's outputs are not checked during prepareModel.
|
||||
if (operation.type == OperationType::SPLIT) {
|
||||
for (const size_t outOprand : operation.outputs) {
|
||||
if (operand == outOprand) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// BIDIRECTIONAL_SEQUENCE_LSTM and BIDIRECTIONAL_SEQUENCE_RNN can have either one or two
|
||||
// outputs depending on their mergeOutputs parameter.
|
||||
if (operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_LSTM ||
|
||||
operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_RNN) {
|
||||
for (const size_t outOprand : operation.outputs) {
|
||||
if (operand == outOprand) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void removeOperandTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
|
||||
if (removeOperandSkip(operand, model)) {
|
||||
continue;
|
||||
}
|
||||
const std::string message = "removeOperandTest: operand " + std::to_string(operand);
|
||||
validate(device, message, model,
|
||||
[operand](Model* model) { removeOperand(model, operand); });
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE OPERATION /////////////////////////
|
||||
|
||||
static void removeOperation(Model* model, uint32_t index) {
|
||||
for (uint32_t operand : model->operations[index].inputs) {
|
||||
model->operands[operand].numberOfConsumers--;
|
||||
}
|
||||
hidl_vec_removeAt(&model->operations, index);
|
||||
}
|
||||
|
||||
static void removeOperationTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
const std::string message = "removeOperationTest: operation " + std::to_string(operation);
|
||||
validate(device, message, model,
|
||||
[operation](Model* model) { removeOperation(model, operation); });
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE OPERATION INPUT /////////////////////////
|
||||
|
||||
static bool removeOperationInputSkip(const Operation& op, size_t input) {
|
||||
// Skip removeOperationInputTest for the following operations.
|
||||
// - CONCATENATION has at least 2 inputs, with the last element being INT32.
|
||||
// - CONV_2D, DEPTHWISE_CONV_2D, MAX_POOL_2D, AVERAGE_POOL_2D, L2_POOL_2D, RESIZE_BILINEAR,
|
||||
// SPACE_TO_DEPTH, SPACE_TO_DEPTH, SPACE_TO_BATCH_ND, BATCH_TO_SPACE_ND can have an optional
|
||||
// layout parameter.
|
||||
// - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis
|
||||
// parameter.
|
||||
switch (op.type) {
|
||||
case OperationType::CONCATENATION: {
|
||||
if (op.inputs.size() > 2 && input != op.inputs.size() - 1) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::DEPTHWISE_CONV_2D: {
|
||||
if ((op.inputs.size() == 12 && input == 11) || (op.inputs.size() == 9 && input == 8)) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::CONV_2D:
|
||||
case OperationType::AVERAGE_POOL_2D:
|
||||
case OperationType::MAX_POOL_2D:
|
||||
case OperationType::L2_POOL_2D: {
|
||||
if ((op.inputs.size() == 11 && input == 10) || (op.inputs.size() == 8 && input == 7)) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::RESIZE_BILINEAR: {
|
||||
if (op.inputs.size() == 4 && input == 3) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::SPACE_TO_DEPTH:
|
||||
case OperationType::DEPTH_TO_SPACE:
|
||||
case OperationType::BATCH_TO_SPACE_ND: {
|
||||
if (op.inputs.size() == 3 && input == 2) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::SPACE_TO_BATCH_ND: {
|
||||
if (op.inputs.size() == 4 && input == 3) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::L2_NORMALIZATION: {
|
||||
if (op.inputs.size() == 2 && input == 1) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::LOCAL_RESPONSE_NORMALIZATION: {
|
||||
if (op.inputs.size() == 6 && input == 5) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
case OperationType::SOFTMAX: {
|
||||
if (op.inputs.size() == 3 && input == 2) {
|
||||
return true;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void removeOperationInputTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
|
||||
const Operation& op = model.operations[operation];
|
||||
if (removeOperationInputSkip(op, input)) {
|
||||
continue;
|
||||
}
|
||||
const std::string message = "removeOperationInputTest: operation " +
|
||||
std::to_string(operation) + ", input " +
|
||||
std::to_string(input);
|
||||
validate(device, message, model, [operation, input](Model* model) {
|
||||
uint32_t operand = model->operations[operation].inputs[input];
|
||||
model->operands[operand].numberOfConsumers--;
|
||||
hidl_vec_removeAt(&model->operations[operation].inputs, input);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE OPERATION OUTPUT /////////////////////////
|
||||
|
||||
static void removeOperationOutputTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
|
||||
const std::string message = "removeOperationOutputTest: operation " +
|
||||
std::to_string(operation) + ", output " +
|
||||
std::to_string(output);
|
||||
validate(device, message, model, [operation, output](Model* model) {
|
||||
hidl_vec_removeAt(&model->operations[operation].outputs, output);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// MODEL VALIDATION /////////////////////////
|
||||
|
||||
// TODO: remove model input
|
||||
// TODO: remove model output
|
||||
// TODO: add unused operation
|
||||
|
||||
///////////////////////// ADD OPERATION INPUT /////////////////////////
|
||||
|
||||
static bool addOperationInputSkip(const Operation& op) {
|
||||
// Skip addOperationInputTest for the following operations.
|
||||
// - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional INT32 axis
|
||||
// parameter.
|
||||
if ((op.type == OperationType::L2_NORMALIZATION && op.inputs.size() == 1) ||
|
||||
(op.type == OperationType::LOCAL_RESPONSE_NORMALIZATION && op.inputs.size() == 5) ||
|
||||
(op.type == OperationType::SOFTMAX && op.inputs.size() == 2)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void addOperationInputTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
if (addOperationInputSkip(model.operations[operation])) {
|
||||
continue;
|
||||
}
|
||||
const std::string message = "addOperationInputTest: operation " + std::to_string(operation);
|
||||
validate(device, message, model, [operation](Model* model) {
|
||||
uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT);
|
||||
hidl_vec_push_back(&model->operations[operation].inputs, index);
|
||||
hidl_vec_push_back(&model->inputIndexes, index);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// ADD OPERATION OUTPUT /////////////////////////
|
||||
|
||||
static void addOperationOutputTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
|
||||
const std::string message =
|
||||
"addOperationOutputTest: operation " + std::to_string(operation);
|
||||
validate(device, message, model, [operation](Model* model) {
|
||||
uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT);
|
||||
hidl_vec_push_back(&model->operations[operation].outputs, index);
|
||||
hidl_vec_push_back(&model->outputIndexes, index);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// VALIDATE EXECUTION PREFERENCE /////////////////////////
|
||||
|
||||
static const int32_t invalidExecutionPreferences[] = {
|
||||
static_cast<int32_t>(ExecutionPreference::LOW_POWER) - 1, // lower bound
|
||||
static_cast<int32_t>(ExecutionPreference::SUSTAINED_SPEED) + 1, // upper bound
|
||||
};
|
||||
|
||||
static void mutateExecutionPreferenceTest(const sp<IDevice>& device, const Model& model) {
|
||||
for (int32_t preference : invalidExecutionPreferences) {
|
||||
const std::string message =
|
||||
"mutateExecutionPreferenceTest: preference " + std::to_string(preference);
|
||||
validate(
|
||||
device, message, model, [](Model*) {},
|
||||
static_cast<ExecutionPreference>(preference));
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////// ENTRY POINT //////////////////////////////
|
||||
|
||||
void validateModel(const sp<IDevice>& device, const Model& model) {
|
||||
mutateOperandTypeTest(device, model);
|
||||
mutateOperandRankTest(device, model);
|
||||
mutateOperandScaleTest(device, model);
|
||||
mutateOperandZeroPointTest(device, model);
|
||||
mutateOperationOperandTypeTest(device, model);
|
||||
mutateOperationTypeTest(device, model);
|
||||
mutateOperationInputOperandIndexTest(device, model);
|
||||
mutateOperationOutputOperandIndexTest(device, model);
|
||||
removeOperandTest(device, model);
|
||||
removeOperationTest(device, model);
|
||||
removeOperationInputTest(device, model);
|
||||
removeOperationOutputTest(device, model);
|
||||
addOperationInputTest(device, model);
|
||||
addOperationOutputTest(device, model);
|
||||
mutateExecutionPreferenceTest(device, model);
|
||||
}
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
168
neuralnetworks/1.3/vts/functional/ValidateRequest.cpp
Normal file
168
neuralnetworks/1.3/vts/functional/ValidateRequest.cpp
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "neuralnetworks_hidl_hal_test"
|
||||
|
||||
#include "1.0/Utils.h"
|
||||
#include "1.2/Callbacks.h"
|
||||
#include "ExecutionBurstController.h"
|
||||
#include "GeneratedTestHarness.h"
|
||||
#include "TestHarness.h"
|
||||
#include "Utils.h"
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using implementation::ExecutionCallback;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::Request;
|
||||
|
||||
///////////////////////// UTILITY FUNCTIONS /////////////////////////
|
||||
|
||||
static bool badTiming(Timing timing) {
|
||||
return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
|
||||
}
|
||||
|
||||
// Primary validation function. This function will take a valid request, apply a
|
||||
// mutation to it to invalidate the request, then pass it to interface calls
|
||||
// that use the request. Note that the request here is passed by value, and any
|
||||
// mutation to the request does not leave this function.
|
||||
static void validate(const sp<IPreparedModel>& preparedModel, const std::string& message,
|
||||
Request request, const std::function<void(Request*)>& mutation) {
|
||||
mutation(&request);
|
||||
|
||||
// We'd like to test both with timing requested and without timing
|
||||
// requested. Rather than running each test both ways, we'll decide whether
|
||||
// to request timing by hashing the message. We do not use std::hash because
|
||||
// it is not guaranteed stable across executions.
|
||||
char hash = 0;
|
||||
for (auto c : message) {
|
||||
hash ^= c;
|
||||
};
|
||||
MeasureTiming measure = (hash & 1) ? MeasureTiming::YES : MeasureTiming::NO;
|
||||
|
||||
// asynchronous
|
||||
{
|
||||
SCOPED_TRACE(message + " [execute_1_2]");
|
||||
|
||||
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
|
||||
Return<ErrorStatus> executeLaunchStatus =
|
||||
preparedModel->execute_1_2(request, measure, executionCallback);
|
||||
ASSERT_TRUE(executeLaunchStatus.isOk());
|
||||
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
|
||||
|
||||
executionCallback->wait();
|
||||
ErrorStatus executionReturnStatus = executionCallback->getStatus();
|
||||
const auto& outputShapes = executionCallback->getOutputShapes();
|
||||
Timing timing = executionCallback->getTiming();
|
||||
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
|
||||
ASSERT_EQ(outputShapes.size(), 0);
|
||||
ASSERT_TRUE(badTiming(timing));
|
||||
}
|
||||
|
||||
// synchronous
|
||||
{
|
||||
SCOPED_TRACE(message + " [executeSynchronously]");
|
||||
|
||||
Return<void> executeStatus = preparedModel->executeSynchronously(
|
||||
request, measure,
|
||||
[](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing) {
|
||||
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
|
||||
EXPECT_EQ(outputShapes.size(), 0);
|
||||
EXPECT_TRUE(badTiming(timing));
|
||||
});
|
||||
ASSERT_TRUE(executeStatus.isOk());
|
||||
}
|
||||
|
||||
// burst
|
||||
{
|
||||
SCOPED_TRACE(message + " [burst]");
|
||||
|
||||
// create burst
|
||||
std::shared_ptr<::android::nn::ExecutionBurstController> burst =
|
||||
android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true);
|
||||
ASSERT_NE(nullptr, burst.get());
|
||||
|
||||
// create memory keys
|
||||
std::vector<intptr_t> keys(request.pools.size());
|
||||
for (size_t i = 0; i < keys.size(); ++i) {
|
||||
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
|
||||
}
|
||||
|
||||
// execute and verify
|
||||
ErrorStatus error;
|
||||
std::vector<OutputShape> outputShapes;
|
||||
Timing timing;
|
||||
std::tie(error, outputShapes, timing) = burst->compute(request, measure, keys);
|
||||
EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
|
||||
EXPECT_EQ(outputShapes.size(), 0);
|
||||
EXPECT_TRUE(badTiming(timing));
|
||||
|
||||
// additional burst testing
|
||||
if (request.pools.size() > 0) {
|
||||
// valid free
|
||||
burst->freeMemory(keys.front());
|
||||
|
||||
// negative test: invalid free of unknown (blank) memory
|
||||
burst->freeMemory(intptr_t{});
|
||||
|
||||
// negative test: double free of memory
|
||||
burst->freeMemory(keys.front());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE INPUT ////////////////////////////////////
|
||||
|
||||
static void removeInputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
|
||||
for (size_t input = 0; input < request.inputs.size(); ++input) {
|
||||
const std::string message = "removeInput: removed input " + std::to_string(input);
|
||||
validate(preparedModel, message, request,
|
||||
[input](Request* request) { hidl_vec_removeAt(&request->inputs, input); });
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////// REMOVE OUTPUT ////////////////////////////////////
|
||||
|
||||
static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
|
||||
for (size_t output = 0; output < request.outputs.size(); ++output) {
|
||||
const std::string message = "removeOutput: removed Output " + std::to_string(output);
|
||||
validate(preparedModel, message, request,
|
||||
[output](Request* request) { hidl_vec_removeAt(&request->outputs, output); });
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////// ENTRY POINT //////////////////////////////////
|
||||
|
||||
void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request) {
|
||||
removeInputTest(preparedModel, request);
|
||||
removeOutputTest(preparedModel, request);
|
||||
}
|
||||
|
||||
void validateRequestFailure(const sp<IPreparedModel>& preparedModel, const Request& request) {
|
||||
SCOPED_TRACE("Expecting request to fail [executeSynchronously]");
|
||||
Return<void> executeStatus = preparedModel->executeSynchronously(
|
||||
request, MeasureTiming::NO,
|
||||
[](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes, const Timing& timing) {
|
||||
ASSERT_NE(ErrorStatus::NONE, error);
|
||||
EXPECT_EQ(outputShapes.size(), 0);
|
||||
EXPECT_TRUE(badTiming(timing));
|
||||
});
|
||||
ASSERT_TRUE(executeStatus.isOk());
|
||||
}
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
171
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp
Normal file
171
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#define LOG_TAG "neuralnetworks_hidl_hal_test"
|
||||
|
||||
#include "VtsHalNeuralnetworks.h"
|
||||
#include <android-base/logging.h>
|
||||
#include <hidl/ServiceManagement.h>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include "1.0/Callbacks.h"
|
||||
#include "1.0/Utils.h"
|
||||
#include "GeneratedTestHarness.h"
|
||||
#include "TestHarness.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using implementation::PreparedModelCallback;
|
||||
using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
|
||||
using V1_0::ErrorStatus;
|
||||
using V1_0::Request;
|
||||
using V1_1::ExecutionPreference;
|
||||
|
||||
// internal helper function
|
||||
void createPreparedModel(const sp<IDevice>& device, const Model& model,
|
||||
sp<IPreparedModel>* preparedModel) {
|
||||
ASSERT_NE(nullptr, preparedModel);
|
||||
*preparedModel = nullptr;
|
||||
|
||||
// see if service can handle model
|
||||
bool fullySupportsModel = false;
|
||||
const Return<void> supportedCall = device->getSupportedOperations_1_2(
|
||||
model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
|
||||
ASSERT_EQ(ErrorStatus::NONE, status);
|
||||
ASSERT_NE(0ul, supported.size());
|
||||
fullySupportsModel = std::all_of(supported.begin(), supported.end(),
|
||||
[](bool valid) { return valid; });
|
||||
});
|
||||
ASSERT_TRUE(supportedCall.isOk());
|
||||
|
||||
// launch prepare model
|
||||
const sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
|
||||
const Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
|
||||
model, ExecutionPreference::FAST_SINGLE_ANSWER, hidl_vec<hidl_handle>(),
|
||||
hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
|
||||
ASSERT_TRUE(prepareLaunchStatus.isOk());
|
||||
ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
|
||||
|
||||
// retrieve prepared model
|
||||
preparedModelCallback->wait();
|
||||
const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
|
||||
*preparedModel = getPreparedModel_1_2(preparedModelCallback);
|
||||
|
||||
// The getSupportedOperations_1_2 call returns a list of operations that are
|
||||
// guaranteed not to fail if prepareModel_1_2 is called, and
|
||||
// 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.
|
||||
// If a driver has any doubt that it can prepare an operation, it must
|
||||
// return false. So here, if a driver isn't sure if it can support an
|
||||
// operation, but reports that it successfully prepared the model, the test
|
||||
// can continue.
|
||||
if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
|
||||
ASSERT_EQ(nullptr, preparedModel->get());
|
||||
LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare "
|
||||
"model that it does not support.";
|
||||
std::cout << "[ ] Early termination of test because vendor service cannot "
|
||||
"prepare model that it does not support."
|
||||
<< std::endl;
|
||||
GTEST_SKIP();
|
||||
}
|
||||
ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);
|
||||
ASSERT_NE(nullptr, preparedModel->get());
|
||||
}
|
||||
|
||||
void NeuralnetworksHidlTest::SetUp() {
|
||||
testing::TestWithParam<NeuralnetworksHidlTestParam>::SetUp();
|
||||
ASSERT_NE(kDevice, nullptr);
|
||||
}
|
||||
|
||||
static NamedDevice makeNamedDevice(const std::string& name) {
|
||||
return {name, IDevice::getService(name)};
|
||||
}
|
||||
|
||||
static std::vector<NamedDevice> getNamedDevicesImpl() {
|
||||
// Retrieves the name of all service instances that implement IDevice,
|
||||
// including any Lazy HAL instances.
|
||||
const std::vector<std::string> names = hardware::getAllHalInstanceNames(IDevice::descriptor);
|
||||
|
||||
// Get a handle to each device and pair it with its name.
|
||||
std::vector<NamedDevice> namedDevices;
|
||||
namedDevices.reserve(names.size());
|
||||
std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice);
|
||||
return namedDevices;
|
||||
}
|
||||
|
||||
const std::vector<NamedDevice>& getNamedDevices() {
|
||||
const static std::vector<NamedDevice> devices = getNamedDevicesImpl();
|
||||
return devices;
|
||||
}
|
||||
|
||||
std::string printNeuralnetworksHidlTest(
|
||||
const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info) {
|
||||
return gtestCompliantName(getName(info.param));
|
||||
}
|
||||
|
||||
INSTANTIATE_DEVICE_TEST(NeuralnetworksHidlTest);
|
||||
|
||||
// Forward declaration from ValidateModel.cpp
|
||||
void validateModel(const sp<IDevice>& device, const Model& model);
|
||||
// Forward declaration from ValidateRequest.cpp
|
||||
void validateRequest(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
|
||||
// Forward declaration from ValidateRequest.cpp
|
||||
void validateRequestFailure(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
|
||||
// Forward declaration from ValidateBurst.cpp
|
||||
void validateBurst(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
|
||||
|
||||
void validateEverything(const sp<IDevice>& device, const Model& model, const Request& request) {
|
||||
validateModel(device, model);
|
||||
|
||||
// Create IPreparedModel.
|
||||
sp<IPreparedModel> preparedModel;
|
||||
createPreparedModel(device, model, &preparedModel);
|
||||
if (preparedModel == nullptr) return;
|
||||
|
||||
validateRequest(preparedModel, request);
|
||||
validateBurst(preparedModel, request);
|
||||
}
|
||||
|
||||
void validateFailure(const sp<IDevice>& device, const Model& model, const Request& request) {
|
||||
// TODO: Should this always succeed?
|
||||
// What if the invalid input is part of the model (i.e., a parameter).
|
||||
validateModel(device, model);
|
||||
|
||||
// Create IPreparedModel.
|
||||
sp<IPreparedModel> preparedModel;
|
||||
createPreparedModel(device, model, &preparedModel);
|
||||
if (preparedModel == nullptr) return;
|
||||
|
||||
validateRequestFailure(preparedModel, request);
|
||||
}
|
||||
|
||||
TEST_P(ValidationTest, Test) {
|
||||
const Model model = createModel(kTestModel);
|
||||
const Request request = createRequest(kTestModel);
|
||||
if (kTestModel.expectFailure) {
|
||||
validateFailure(kDevice, model, request);
|
||||
} else {
|
||||
validateEverything(kDevice, model, request);
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; });
|
||||
|
||||
sp<IPreparedModel> getPreparedModel_1_2(const sp<implementation::PreparedModelCallback>& callback) {
|
||||
sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel();
|
||||
return IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);
|
||||
}
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
57
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h
Normal file
57
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H
|
||||
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H
|
||||
|
||||
#include <android/hardware/neuralnetworks/1.2/IDevice.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/types.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include "1.0/Utils.h"
|
||||
#include "1.2/Callbacks.h"
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::vts::functional {
|
||||
|
||||
using NamedDevice = Named<sp<IDevice>>;
|
||||
using NeuralnetworksHidlTestParam = NamedDevice;
|
||||
|
||||
class NeuralnetworksHidlTest : public testing::TestWithParam<NeuralnetworksHidlTestParam> {
|
||||
protected:
|
||||
void SetUp() override;
|
||||
const sp<IDevice> kDevice = getData(GetParam());
|
||||
};
|
||||
|
||||
const std::vector<NamedDevice>& getNamedDevices();
|
||||
|
||||
std::string printNeuralnetworksHidlTest(
|
||||
const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info);
|
||||
|
||||
#define INSTANTIATE_DEVICE_TEST(TestSuite) \
|
||||
INSTANTIATE_TEST_SUITE_P(PerInstance, TestSuite, testing::ValuesIn(getNamedDevices()), \
|
||||
printNeuralnetworksHidlTest)
|
||||
|
||||
// Create an IPreparedModel object. If the model cannot be prepared,
|
||||
// "preparedModel" will be nullptr instead.
|
||||
void createPreparedModel(const sp<IDevice>& device, const Model& model,
|
||||
sp<IPreparedModel>* preparedModel);
|
||||
|
||||
// Utility function to get PreparedModel from callback and downcast to V1_2.
|
||||
sp<IPreparedModel> getPreparedModel_1_2(const sp<implementation::PreparedModelCallback>& callback);
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::vts::functional
|
||||
|
||||
#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_VTS_HAL_NEURALNETWORKS_H
|
||||
325
neuralnetworks/1.3/vts/functional/include/1.2/Callbacks.h
Normal file
325
neuralnetworks/1.3/vts/functional/include/1.2/Callbacks.h
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
/*
|
||||
* Copyright (C) 2018 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H
|
||||
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H
|
||||
|
||||
#include <android-base/thread_annotations.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
|
||||
#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
|
||||
#include <hidl/Status.h>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
|
||||
/*
|
||||
* The Callback classes are used internally by the NeuralNetworks runtime to
|
||||
* synchronize between different threads. An asynchronous task is launched
|
||||
* paired with a callback object. When a client thread requires the output being
|
||||
* generated by the asynchronous task, the client thread can wait for the result
|
||||
* and be blocked until it has completed. Any wait may safely be called
|
||||
* concurrently, even on the same callback object. When the asynchronous task
|
||||
* has finished its workload, it must immediately call "notify*". If the
|
||||
* asynchronous task has failed to launch, the function that tried to launch the
|
||||
* asynchronous task must immediately call "notify*". This "notify*" call
|
||||
* awakens any client threads waiting on the callback object.
|
||||
*
|
||||
* These classes exist to enable synchronization across HIDL. When
|
||||
* synchronization is only required in the same process, consider using
|
||||
* std::future, std::mutex, std::condition_variable, or std::experimental::latch
|
||||
* instead.
|
||||
*/
|
||||
|
||||
namespace android::hardware::neuralnetworks::V1_2::implementation {
|
||||
|
||||
/**
|
||||
* The PreparedModelCallback class is used to receive the error status of
|
||||
* preparing a model as well as the prepared model from a task executing
|
||||
* asynchronously with respect to the runtime. If a calling thread calls wait
|
||||
* or get* on a PreparedModelCallback object and the corresponding asynchronous
|
||||
* task has not finished preparing the model, the calling thread will block
|
||||
* until the asynchronous task has either called notify or notify_1_2.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of the
|
||||
* first call to notify* are used, and the results from subsequent calls are
|
||||
* discarded.
|
||||
*
|
||||
* This callback object is passed as an argument to IDevice::prepareModel*.
|
||||
*/
|
||||
class PreparedModelCallback : public IPreparedModelCallback {
|
||||
public:
|
||||
/**
|
||||
* IPreparedModelCallback::notify marks the callback object with the return
|
||||
* status of the asynchronous model preparation along with the prepared
|
||||
* model, and allows all prior and future wait calls on the
|
||||
* PreparedModelCallback object to proceed.
|
||||
*
|
||||
* Either IPreparedModelCallback::notify or
|
||||
* IPreparedModelCallback::notify_1_2 must be called on a given
|
||||
* PreparedModelCallback object.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of
|
||||
* the first call to notify* are used, and the results from subsequent calls
|
||||
* are discarded.
|
||||
*
|
||||
* @param status Error status returned from asynchronously preparing the
|
||||
* model; will be:
|
||||
* - NONE if the asynchronous preparation was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if there is an unspecified error
|
||||
* - INVALID_ARGUMENT if the input model is invalid
|
||||
* @param preparedModel Returned model that has been prepared for execution,
|
||||
* nullptr if the model was unable to be prepared.
|
||||
*/
|
||||
Return<void> notify(V1_0::ErrorStatus status,
|
||||
const sp<V1_0::IPreparedModel>& preparedModel) override;
|
||||
|
||||
/**
|
||||
* IPreparedModelCallback::notify_1_2 marks the callback object with the
|
||||
* return status of the asynchronous model preparation along with the
|
||||
* prepared model, and allows all prior and future wait calls on the
|
||||
* PreparedModelCallback object to proceed.
|
||||
*
|
||||
* Either IPreparedModelCallback::notify or
|
||||
* IPreparedModelCallback::notify_1_2 must be called on a given
|
||||
* PreparedModelCallback object.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of
|
||||
* the first call to notify* are used, and the results from subsequent calls
|
||||
* are discarded.
|
||||
*
|
||||
* @param status Error status returned from asynchronously preparing the
|
||||
* model; will be:
|
||||
* - NONE if the asynchronous preparation was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if there is an unspecified error
|
||||
* - INVALID_ARGUMENT if the input model is invalid
|
||||
* @param preparedModel Returned model that has been prepared for execution,
|
||||
* nullptr if the model was unable to be prepared.
|
||||
*/
|
||||
Return<void> notify_1_2(V1_0::ErrorStatus status,
|
||||
const sp<V1_2::IPreparedModel>& preparedModel) override;
|
||||
|
||||
/**
|
||||
* PreparedModelCallback::wait blocks until notify* has been called on the
|
||||
* callback object.
|
||||
*/
|
||||
void wait() const;
|
||||
|
||||
/**
|
||||
* Retrieves the error status returned from the asynchronous task launched
|
||||
* by IDevice::prepareModel*. If IDevice::prepareModel* has not finished
|
||||
* asynchronously preparing the model, this call will block until the
|
||||
* asynchronous task notifies the object.
|
||||
*
|
||||
* @return status Error status returned from asynchronously preparing the
|
||||
* model; will be:
|
||||
* - NONE if the asynchronous preparation was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if there is an unspecified error
|
||||
* - INVALID_ARGUMENT if the input model is invalid
|
||||
*/
|
||||
V1_0::ErrorStatus getStatus() const;
|
||||
|
||||
/**
|
||||
* Retrieves the model that has been prepared for execution from the
|
||||
* asynchronous task launched by IDevice::prepareModel*. If
|
||||
* IDevice::prepareModel* has not finished asynchronously preparing the
|
||||
* model, this call will block until the asynchronous task notifies the
|
||||
* object.
|
||||
*
|
||||
* @return preparedModel Returned model that has been prepared for
|
||||
* execution, nullptr if the model was unable to be prepared.
|
||||
*/
|
||||
sp<V1_0::IPreparedModel> getPreparedModel() const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mMutex;
|
||||
mutable std::condition_variable mCondition;
|
||||
bool mNotified GUARDED_BY(mMutex) = false;
|
||||
V1_0::ErrorStatus mErrorStatus = V1_0::ErrorStatus::GENERAL_FAILURE;
|
||||
sp<V1_0::IPreparedModel> mPreparedModel;
|
||||
};
|
||||
|
||||
/**
|
||||
* The ExecutionCallback class is used to receive the results of the execution
|
||||
* from a task executing asynchronously with respect to the runtime. If a
|
||||
* calling thread calls wait or get* on a ExecutionCallback object and the
|
||||
* corresponding asynchronous task has not finished the execution, the calling
|
||||
* thread will block until the asynchronous task has either called notify or
|
||||
* notify_1_2.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of the
|
||||
* first call to notify* are used, and the results from subsequent calls are
|
||||
* discarded.
|
||||
*
|
||||
* This callback object is passed as an argument to IPreparedModel::execute*.
|
||||
*/
|
||||
class ExecutionCallback : public IExecutionCallback {
|
||||
public:
|
||||
/**
|
||||
* IExecutionCallback::notify marks the callback object with the return
|
||||
* status of the asynchronous execution that held this callback and enables
|
||||
* all prior and future wait calls on the ExecutionCallback object to
|
||||
* proceed.
|
||||
*
|
||||
* Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must
|
||||
* be called on a given ExecutionCallback object.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of
|
||||
* the first call to notify* are used, and the results from subsequent calls
|
||||
* are discarded.
|
||||
*
|
||||
* @param status Error status returned from launching the asynchronous task
|
||||
* (if the launch fails) or from the asynchronous task itself (if the
|
||||
* launch succeeds). Must be:
|
||||
* - NONE if the asynchronous execution was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if there is an unspecified error
|
||||
* - OUTPUT_INSUFFICIENT_SIZE if provided output buffer is not large
|
||||
* enough to store the resultant values
|
||||
* - INVALID_ARGUMENT if the input request is invalid
|
||||
*/
|
||||
Return<void> notify(V1_0::ErrorStatus status) override;
|
||||
|
||||
/**
|
||||
* IExecutionCallback::notify_1_2 marks the callback object with the results
|
||||
* (error status, dynamic output shapes, and timing information) of the
|
||||
* asynchronous execution that held this callback and enables all prior and
|
||||
* future wait calls on the ExecutionCallback object to proceed.
|
||||
*
|
||||
* Either IExecutionCallback::notify or IExecutionCallback::notify_1_2 must
|
||||
* be called on a given ExecutionCallback object.
|
||||
*
|
||||
* If the callback object is notified more than once, only the results of
|
||||
* the first call to notify* are used, and the results from subsequent calls
|
||||
* are discarded.
|
||||
*
|
||||
* @param status Error status returned from launching the asynchronous task
|
||||
* (if the launch fails) or from the asynchronous task itself (if the
|
||||
* launch succeeds). Must be:
|
||||
* - NONE if the asynchronous execution was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if the asynchronous task resulted in an unspecified
|
||||
* error
|
||||
* - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is
|
||||
* not large enough to store the corresponding output
|
||||
* - INVALID_ARGUMENT if one of the input arguments to prepareModel is
|
||||
* invalid
|
||||
* @param outputShapes A list of shape information of model output operands.
|
||||
* The index into "outputShapes" corresponds to the index of the output
|
||||
* operand in the Request outputs vector. outputShapes must be empty
|
||||
* unless the status is either NONE or OUTPUT_INSUFFICIENT_SIZE.
|
||||
* @param Timing Duration of execution. Unless MeasureTiming::YES was passed
|
||||
* when launching the execution and status is NONE, all times must be
|
||||
* reported as UINT64_MAX. A driver may choose to report any time as
|
||||
* UINT64_MAX, indicating that particular measurement is not available.
|
||||
*/
|
||||
Return<void> notify_1_2(V1_0::ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing) override;
|
||||
|
||||
// An overload of the latest notify interface to hide the version from ExecutionBuilder.
|
||||
Return<void> notify(V1_0::ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing) {
|
||||
return notify_1_2(status, outputShapes, timing);
|
||||
}
|
||||
|
||||
/**
|
||||
* ExecutionCallback::wait blocks until notify* has been called on the
|
||||
* callback object.
|
||||
*/
|
||||
void wait() const;
|
||||
|
||||
/**
|
||||
* Retrieves the error status returned from the asynchronous task launched
|
||||
* by either IPreparedModel::execute or IPreparedModel::execute_1_2. If
|
||||
* IPreparedModel::execute or IPreparedModel::execute_1_2 has not finished
|
||||
* asynchronously executing, this call will block until the asynchronous
|
||||
* task notifies the object.
|
||||
*
|
||||
* @return status Error status returned from launching the asynchronous task
|
||||
* (if the launch fails) or from the asynchronous task itself (if the
|
||||
* launch succeeds). Must be:
|
||||
* - NONE if the asynchronous execution was successful
|
||||
* - DEVICE_UNAVAILABLE if driver is offline or busy
|
||||
* - GENERAL_FAILURE if the asynchronous task resulted in an unspecified
|
||||
* error
|
||||
* - OUTPUT_INSUFFICIENT_SIZE if at least one output operand buffer is
|
||||
* not large enough to store the corresponding output
|
||||
* - INVALID_ARGUMENT if one of the input arguments to prepareModel is
|
||||
* invalid
|
||||
*/
|
||||
V1_0::ErrorStatus getStatus() const;
|
||||
|
||||
/**
|
||||
* Retrieves the output shapes returned from the asynchronous task launched
|
||||
* by IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not
|
||||
* finished asynchronously executing, this call will block until the
|
||||
* asynchronous task notifies the object.
|
||||
*
|
||||
* If the asynchronous task was launched by IPreparedModel::execute, an
|
||||
* empty vector will be returned.
|
||||
*
|
||||
* @return outputShapes A list of shape information of model output
|
||||
* operands. The index into "outputShapes" corresponds to the index of
|
||||
* the output operand in the Request outputs vector. outputShapes must
|
||||
* be empty unless the status is either NONE or
|
||||
* OUTPUT_INSUFFICIENT_SIZE. outputShaps may be empty if the status is
|
||||
* NONE and all model output operands are fully-specified at execution
|
||||
* time. outputShapes must have the same number of elements as the
|
||||
* number of model output operands if the status is
|
||||
* OUTPUT_INSUFFICIENT_SIZE, or if the status is NONE and the model has
|
||||
* at least one output operand that is not fully-specified.
|
||||
*/
|
||||
const std::vector<OutputShape>& getOutputShapes() const;
|
||||
|
||||
/**
|
||||
* Retrieves the duration of execution of the asynchronous task launched by
|
||||
* IPreparedModel::execute_1_2. If IPreparedModel::execute_1_2 has not
|
||||
* finished asynchronously executing, this call will block until the
|
||||
* asynchronous task notifies the object.
|
||||
*
|
||||
* If the asynchronous task was launched by IPreparedModel::execute, every
|
||||
* time must be UINT64_MAX.
|
||||
*
|
||||
* @return timing Duration of the execution. Every time must be UINT64_MAX
|
||||
* unless the status is NONE.
|
||||
*/
|
||||
Timing getTiming() const;
|
||||
|
||||
private:
|
||||
/*
|
||||
* ExecutionCallback::notifyInternal stores the results of the execution
|
||||
* (status, output shapes, and timing information) in the ExecutionCallback
|
||||
* object before any call to wait or get* return. It then enables all prior
|
||||
* and future wait calls on the ExecutionCallback object to proceed.
|
||||
*/
|
||||
void notifyInternal(V1_0::ErrorStatus errorStatus, const hidl_vec<OutputShape>& outputShapes,
|
||||
const Timing& timing);
|
||||
|
||||
// members
|
||||
mutable std::mutex mMutex;
|
||||
mutable std::condition_variable mCondition;
|
||||
bool mNotified GUARDED_BY(mMutex) = false;
|
||||
V1_0::ErrorStatus mErrorStatus = V1_0::ErrorStatus::GENERAL_FAILURE;
|
||||
std::vector<OutputShape> mOutputShapes = {};
|
||||
Timing mTiming = {};
|
||||
};
|
||||
|
||||
} // namespace android::hardware::neuralnetworks::V1_2::implementation
|
||||
|
||||
#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_2_CALLBACKS_H
|
||||
Loading…
Reference in a new issue