tequilaOS/platform_hardware_interfaces
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge changes from topic "NNAPI v1.3"

Browse source 
* changes:
  Revert "Create NNAPI HAL v1.3 and add TENSOR_QUANT8_ASYMM_SIGNED OperandType"
  Revert "Copy VTS tests from v1.2 to v1.3"
  Revert "Modify NNAPI VTS tests to run on version 1.3"
This commit is contained in:
Lev Proleev 2019-10-10 14:06:42 +00:00 committed by Gerrit Code Review
commit b8d9568f3d
17 changed files with 1 additions and 4245 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
current.txt
View file

@ -586,5 +586,3 @@ fd65298e1e09e0e3c781ab18305920d757dbe55a3b459ce17814ec5cf6dfee99 android.hardwar
# HALs released in Android R
07d0a252b2d8fa35887908a996ba395cf392968395fc30afab791f46e0c22a52 android.hardware.boot@1.1::IBootControl
74049a402be913963edfdd80828a53736570e9d8124a1bf18166b6ed46a6b0ab android.hardware.boot@1.1::types
34515afa2bb792d3c6d8495a5f5d907d179c8507ca5e55c10050d02ae1d516ef android.hardware.neuralnetworks@1.3::IDevice
e2d20d4eb24f40b44a3766d05f77052581cb3f4df35fb48c0cc5d9cdcf5c872e android.hardware.neuralnetworks@1.3::types

20
neuralnetworks/1.2/vts/functional/Android.bp
View file

@ -14,28 +14,12 @@
// limitations under the License.
//
cc_library_static {
name: "VtsHalNeuralNetworksV1_2Callbacks",
defaults: ["VtsHalTargetTestDefaults"],
export_include_dirs: ["include"],
srcs: [
"Callbacks.cpp",
],
static_libs: [
"android.hardware.neuralnetworks@1.0",
"android.hardware.neuralnetworks@1.1",
"android.hardware.neuralnetworks@1.2",
],
header_libs: [
"libbase_headers",
]
}
cc_test {
name: "VtsHalNeuralnetworksV1_2TargetTest",
defaults: ["VtsHalTargetTestDefaults"],
srcs: [
"BasicTests.cpp",
"Callbacks.cpp",
"CompilationCachingTests.cpp",
"GeneratedTestHarness.cpp",
"TestAssertions.cpp",
@ -53,7 +37,6 @@ cc_test {
"android.hardware.neuralnetworks@1.0",
"android.hardware.neuralnetworks@1.1",
"android.hardware.neuralnetworks@1.2",
"android.hardware.neuralnetworks@1.3",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
"libgmock",
@ -61,7 +44,6 @@ cc_test {
"libneuralnetworks_generated_test_harness",
"libneuralnetworks_utils",
"VtsHalNeuralNetworksV1_0_utils",
"VtsHalNeuralNetworksV1_2Callbacks",
],
whole_static_libs: [
"neuralnetworks_generated_V1_0_example",

21
neuralnetworks/1.3/Android.bp
View file

@ -1,21 +0,0 @@
// This file is autogenerated by hidl-gen -Landroidbp.
hidl_interface {
name: "android.hardware.neuralnetworks@1.3",
root: "android.hardware",
vndk: {
enabled: true,
},
srcs: [
"types.hal",
"IDevice.hal",
],
interfaces: [
"android.hardware.neuralnetworks@1.0",
"android.hardware.neuralnetworks@1.1",
"android.hardware.neuralnetworks@1.2",
"android.hidl.base@1.0",
"android.hidl.safe_union@1.0",
],
gen_java: false,
}

171
neuralnetworks/1.3/IDevice.hal
View file

@ -1,171 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.hardware.neuralnetworks@1.3;
import @1.0::ErrorStatus;
import @1.1::ExecutionPreference;
import @1.2::Constant;
import @1.2::DeviceType;
import @1.2::Extension;
import @1.2::IDevice;
import @1.2::IPreparedModelCallback;
/**
* This interface represents a device driver.
*/
interface IDevice extends @1.2::IDevice {
/**
* Gets the capabilities of a driver.
*
* @return status Error status of the call, must be:
* - NONE if successful
* - DEVICE_UNAVAILABLE if driver is offline or busy
* - GENERAL_FAILURE if there is an unspecified error
* @return capabilities Capabilities of the driver.
*/
getCapabilities_1_3() generates (ErrorStatus status, Capabilities capabilities);
/**
* Gets the supported operations in a model.
*
* getSupportedOperations indicates which operations of a model are fully
* supported by the vendor driver. If an operation may not be supported for
* any reason, getSupportedOperations must return false for that operation.
*
* @param model A model whose operations--and their corresponding operands--
* are to be verified by the driver.
* @return status Error status of the call, must be:
* - NONE if successful
* - DEVICE_UNAVAILABLE if driver is offline or busy
* - GENERAL_FAILURE if there is an unspecified error
* - INVALID_ARGUMENT if provided model is invalid
* @return supportedOperations A list of supported operations, where true
* indicates the operation is supported and false indicates the
* operation is not supported. The index of "supported" corresponds with
* the index of the operation it is describing.
*/
getSupportedOperations_1_3(Model model)
generates (ErrorStatus status, vec<bool> supportedOperations);
/**
* Asynchronously creates a prepared model for execution and optionally
* saves it into cache files.
*
* prepareModel is used to make any necessary transformations to or
* alternative representations to a model for execution, possibly including
* transformations on the constant data, optimization on the model's graph,
* or compilation into the device's native binary format. The model itself
* is not changed.
*
* Optionally, caching information may be provided for the driver to save
* the prepared model to cache files for faster model compilation time when
* the same model preparation is requested in the future. There are two
* types of cache file handles provided to the driver: model cache and data
* cache. For more information on the two types of cache handles, refer to
* getNumberOfCacheFilesNeeded.
*
* The file descriptors must be opened with read and write permission. A
* file may have any size, and the corresponding file descriptor may have
* any offset. The driver must truncate a file to zero size before writing
* to that file. The file descriptors may be closed by the client once the
* asynchronous preparation has finished. The driver must dup a file
* descriptor if it wants to get access to the cache file later.
*
* The model is prepared asynchronously with respect to the caller. The
* prepareModel function must verify the inputs to the preparedModel
* function related to preparing the model (as opposed to saving the
* prepared model to cache) are correct. If there is an error, prepareModel
* must immediately invoke the callback with the appropriate ErrorStatus
* value and nullptr for the IPreparedModel, then return with the same
* ErrorStatus. If the inputs to the prepareModel function that are related
* to preparing the model are valid and there is no error, prepareModel must
* launch an asynchronous task to prepare the model in the background, and
* immediately return from prepareModel with ErrorStatus::NONE. If the
* asynchronous task fails to launch, prepareModel must immediately invoke
* the callback with ErrorStatus::GENERAL_FAILURE and nullptr for the
* IPreparedModel, then return with ErrorStatus::GENERAL_FAILURE.
*
* When the asynchronous task has finished preparing the model, it must
* immediately invoke the callback function provided as an input to
* prepareModel. If the model was prepared successfully, the callback object
* must be invoked with an error status of ErrorStatus::NONE and the
* produced IPreparedModel object. If an error occurred preparing the model,
* the callback object must be invoked with the appropriate ErrorStatus
* value and nullptr for the IPreparedModel.
*
* Optionally, the driver may save the prepared model to cache during the
* asynchronous preparation. Any error that occurs when saving to cache must
* not affect the status of preparing the model. Even if the input arguments
* related to the cache may be invalid, or the driver may fail to save to
* cache, the prepareModel function must finish preparing the model. The
* driver may choose not to save to cache even if the caching information is
* provided and valid.
*
* The only information that may be unknown to the model at this stage is
* the shape of the tensors, which may only be known at execution time. As
* such, some driver services may return partially prepared models, where
* the prepared model may only be finished when it is paired with a set of
* inputs to the model. Note that the same prepared model object may be used
* with different shapes of inputs on different (possibly concurrent)
* executions.
*
* Multiple threads may call prepareModel on the same model concurrently.
*
* @param model The model to be prepared for execution.
* @param preference Indicates the intended execution behavior of a prepared
* model.
* @param modelCache A vector of handles with each entry holding exactly one
* cache file descriptor for the security-sensitive cache. The length of
* the vector must either be 0 indicating that caching information is
* not provided, or match the numModelCache returned from
* getNumberOfCacheFilesNeeded. The cache handles will be provided in
* the same order when retrieving the preparedModel from cache files
* with prepareModelFromCache.
* @param dataCache A vector of handles with each entry holding exactly one
* cache file descriptor for the constants' cache. The length of the
* vector must either be 0 indicating that caching information is not
* provided, or match the numDataCache returned from
* getNumberOfCacheFilesNeeded. The cache handles will be provided in
* the same order when retrieving the preparedModel from cache files
* with prepareModelFromCache.
* @param token A caching token of length Constant::BYTE_SIZE_OF_CACHE_TOKEN
* identifying the prepared model. The same token will be provided when
* retrieving the prepared model from the cache files with
* prepareModelFromCache. Tokens should be chosen to have a low rate of
* collision for a particular application. The driver cannot detect a
* collision; a collision will result in a failed execution or in a
* successful execution that produces incorrect output values. If both
* modelCache and dataCache are empty indicating that caching
* information is not provided, this token must be ignored.
* @param callback A callback object used to return the error status of
* preparing the model for execution and the prepared model if
* successful, nullptr otherwise. The callback object's notify function
* must be called exactly once, even if the model could not be prepared.
* @return status Error status of launching a task which prepares the model
* in the background; must be:
* - NONE if preparation task is successfully launched
* - DEVICE_UNAVAILABLE if driver is offline or busy
* - GENERAL_FAILURE if there is an unspecified error
* - INVALID_ARGUMENT if one of the input arguments related to preparing
* the model is invalid
*/
prepareModel_1_3(Model model, ExecutionPreference preference,
vec<handle> modelCache, vec<handle> dataCache,
uint8_t[Constant:BYTE_SIZE_OF_CACHE_TOKEN] token,
IPreparedModelCallback callback)
generates (ErrorStatus status);
};

361
neuralnetworks/1.3/types.hal
View file

@ -1,361 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.hardware.neuralnetworks@1.3;
import @1.0::DataLocation;
import @1.0::OperandLifeTime;
import @1.0::PerformanceInfo;
import @1.2::OperandType;
import @1.2::OperationType;
import @1.2::SymmPerChannelQuantParams;
import android.hidl.safe_union@1.0::Monostate;
/**
* NOTE: Since NNAPI 1.2, OEM operation and data type are deprecated. Extensions
* are the preferred alternative.
*
* NOTE: Adding a new fundamental type requires updating the value of
* OperandTypeRange::FUNDAMENTAL_MAX.
*/
enum OperandType : @1.2::OperandType {
/**
* A tensor of 8 bit signed integers that represent real numbers.
*
* Attached to this tensor are two numbers that can be used to convert the
* 8 bit integer to the real value and vice versa. These two numbers are:
* - scale: a 32 bit floating point value greater than zero.
* - zeroPoint: a 32 bit integer, in range [-128, 127].
*
* The formula is:
* real_value = (integer_value - zeroPoint) * scale.
*
* Available since API level 30.
*/
TENSOR_QUANT8_ASYMM_SIGNED = 14,
};
/**
* The range of operand values in the OperandType enum.
*/
enum OperandTypeRange : uint32_t {
BASE_MIN = 0,
FUNDAMENTAL_MIN = 0,
FUNDAMENTAL_MAX = 14,
OEM_MIN = 10000,
OEM_MAX = 10001,
BASE_MAX = 0xFFFF,
};
/**
* The capabilities of a driver.
*
* Performance of an operation comes from the type of its first operand.
* This represents performance for non extension operand types.
*/
struct Capabilities {
/**
* Driver performance when operating on float32 data but performing
* calculations with range and/or precision as low as that of the IEEE
* 754 16-bit floating-point format.
*/
PerformanceInfo relaxedFloat32toFloat16PerformanceScalar;
PerformanceInfo relaxedFloat32toFloat16PerformanceTensor;
/**
* Driver performance when operating on a particular data type.
* In the case of float32 data, this is used when the calculations
* are not relaxed.
*/
struct OperandPerformance {
OperandType type;
PerformanceInfo info;
};
/**
* Performance by operand type. Must be sorted by OperandType.
* If a particular OperandType is not present in operandPerformance,
* its performance is treated as
* { .execTime = FLT_MAX, .powerUsage = FLT_MAX }.
*/
vec<OperandPerformance> operandPerformance;
};
/**
* Describes one operand of the model's graph.
*/
struct Operand {
/**
* The data type.
*
* Besides the values listed in {@link OperandType}, any value above
* {@link OperandTypeRange::BASE_MAX} is possible and should be interpreted
* as an extension type according to {@link Model::extensionNameToPrefix}.
*/
OperandType type;
/**
* Dimensions of the operand.
*
* For a scalar operand, dimensions.size() must be 0.
*
* A tensor operand with all dimensions specified has "fully
* specified" dimensions. Whenever possible (i.e., whenever the
* dimensions are known at model construction time), a tensor
* operand should have (but is not required to have) fully
* specified dimensions, in order to enable the best possible
* performance.
*
* If a tensor operand's dimensions are not fully specified, the
* dimensions of the operand are deduced from the operand
* dimensions and values of the operation for which that operand
* is an output.
*
* In the following situations, a tensor operand's dimensions must
* be fully specified:
*
* . The operand has lifetime CONSTANT_COPY or
* CONSTANT_REFERENCE.
*
* . The operand has lifetime MODEL_INPUT. Fully
* specified dimensions must either be present in the
* Operand or they must be provided in the corresponding
* RequestArgument.
* EXCEPTION: If the input is optional and omitted
* (by setting the hasNoValue field of the corresponding
* RequestArgument to true) then it need not have fully
* specified dimensions.
*
* A tensor operand with some number of unspecified dimensions is
* represented by setting each unspecified dimension to 0.
*
* A tensor operand with unspecified rank is represented by providing
* an empty dimensions vector.
*/
vec<uint32_t> dimensions;
/**
* The number of times this operand appears as an operation input.
*
* (For example, if this operand appears once in one operation's
* input list, and three times in another operation's input list,
* then numberOfConsumers = 4.)
*/
uint32_t numberOfConsumers;
/**
* Quantized scale of the operand.
*
* Only applicable if the operand is of type TENSOR_QUANT8_ASYMM or
* TENSOR_INT32.
*/
float scale;
/**
* Quantized zero-point offset of the operand.
*
* Only applicable if the operand is of type TENSOR_QUANT8_ASYMM.
*/
int32_t zeroPoint;
/**
* How the operand is used.
*/
OperandLifeTime lifetime;
/**
* Where to find the data for this operand.
* If the lifetime is TEMPORARY_VARIABLE, MODEL_INPUT, MODEL_OUTPUT, or
* NO_VALUE:
* - All the fields must be 0.
* If the lifetime is CONSTANT_COPY:
* - location.poolIndex is 0.
* - location.offset is the offset in bytes into Model.operandValues.
* - location.length is set.
* If the lifetime is CONSTANT_REFERENCE:
* - location.poolIndex is set.
* - location.offset is the offset in bytes into the specified pool.
* - location.length is set.
*/
DataLocation location;
/**
* Additional parameters specific to a particular operand type.
*/
safe_union ExtraParams {
/**
* No additional parameters.
*/
Monostate none;
/**
* Symmetric per-channel quantization parameters.
*
* Only applicable to operands of type TENSOR_QUANT8_SYMM_PER_CHANNEL.
*/
SymmPerChannelQuantParams channelQuant;
/**
* Extension operand parameters.
*
* The framework treats this as an opaque data blob.
* The format is up to individual extensions.
*/
vec<uint8_t> extension;
} extraParams;
};
/**
* Describes one operation of the model's graph.
*/
struct Operation {
/**
* The operation type.
*/
OperationType type;
/**
* Describes the table that contains the indexes of the inputs of the
* operation. The offset is the index in the operandIndexes table.
*/
vec<uint32_t> inputs;
/**
* Describes the table that contains the indexes of the outputs of the
* operation. The offset is the index in the operandIndexes table.
*/
vec<uint32_t> outputs;
};
/**
* A Neural Network Model.
*
* This includes not only the execution graph, but also constant data such as
* weights or scalars added at construction time. The only information that
* may not be known is the shape of the input tensors.
*/
struct Model {
/**
* All operands included in the model.
*/
vec<Operand> operands;
/**
* All operations included in the model.
*
* The operations are sorted into execution order. Every operand
* with lifetime MODEL_OUTPUT or TEMPORARY_VARIABLE must be
* written before it is read.
*/
vec<Operation> operations;
/**
* Input indexes of the model. There must be at least one.
*
* Each value corresponds to the index of the operand in "operands".
*/
vec<uint32_t> inputIndexes;
/**
* Output indexes of the model. There must be at least one.
*
* Each value corresponds to the index of the operand in "operands".
*/
vec<uint32_t> outputIndexes;
/**
* A byte buffer containing operand data that were copied into the model.
*
* An operand's value must be located here if and only if Operand::lifetime
* equals OperandLifeTime::CONSTANT_COPY.
*/
vec<uint8_t> operandValues;
/**
* A collection of shared memory pools containing operand values.
*
* An operand's value must be located here if and only if Operand::lifetime
* equals OperandLifeTime::CONSTANT_REFERENCE.
*/
vec<memory> pools;
/**
* 'true' indicates TENSOR_FLOAT32 may be calculated with range and/or
* precision as low as that of the IEEE 754 16-bit floating-point format.
* 'false' indicates TENSOR_FLOAT32 must be calculated using at least the
* range and precision of the IEEE 754 32-bit floating-point format.
*/
bool relaxComputationFloat32toFloat16;
/**
* The mapping between extension names and prefixes of operand and
* operation type values.
*
* An operand or operation whose numeric type value is above
* {@link OperandTypeRange::BASE_MAX} or
* {@link OperationTypeRange::BASE_MAX} respectively should be interpreted
* as an extension operand. The low
* {@link Model::ExtensionTypeEncoding::LOW_BITS_TYPE} bits of the value
* correspond to the type ID within the extension and the high
* {@link Model::ExtensionTypeEncoding::HIGH_BITS_PREFIX} bits encode
* the "prefix", which maps uniquely to the extension name.
*
* For example, if a model contains an operation whose value is
* 0xAAAABBBB and extensionNameToPrefix contains an entry with
* prefix=0xAAAA and name="vendor.test.test_extension", then
* the operation should be interpreted as the operation 0xBBBB
* of the extension named vendor.test.test_extension.
*
* This is a one-to-one correspondence. That is, there must be at most one
* prefix corresponding to each extension name and at most one extension
* name corresponding to each prefix.
*/
vec<ExtensionNameAndPrefix> extensionNameToPrefix;
/**
* A correspondence between an extension name and a prefix of operand and
* operation type values.
*/
struct ExtensionNameAndPrefix {
/**
* The extension name.
*
* See {@link Extension::name} for the format specification.
*/
string name;
/**
* The unique extension identifier within the model.
*
* See {@link Model::extensionNameToPrefix}.
*/
uint16_t prefix;
};
/**
* Numeric values of extension operand and operation types have the
* following structure:
* - 16 high bits represent the "prefix", which corresponds uniquely to the
* extension name.
* - 16 low bits represent the type ID within the extension.
*/
enum ExtensionTypeEncoding : uint8_t {
HIGH_BITS_PREFIX = 16,
LOW_BITS_TYPE = 16,
};
};

16
neuralnetworks/1.3/vts/OWNERS
View file

@ -1,16 +0,0 @@
# Neuralnetworks team
butlermichael@google.com
dgross@google.com
jeanluc@google.com
levp@google.com
miaowang@google.com
mikie@google.com
mks@google.com
pszczepaniak@google.com
slavash@google.com
vddang@google.com
xusongw@google.com
# VTS team
yim@google.com
yuexima@google.com

58
neuralnetworks/1.3/vts/functional/Android.bp
View file

@ -1,58 +0,0 @@
//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
cc_test {
name: "VtsHalNeuralNetworksV1_3TargetTest",
defaults: ["VtsHalTargetTestDefaults"],
srcs: [
"BasicTests.cpp",
"CompilationCachingTests.cpp",
"GeneratedTestHarness.cpp",
"TestAssertions.cpp",
"ValidateBurst.cpp",
"ValidateModel.cpp",
"ValidateRequest.cpp",
"VtsHalNeuralnetworks.cpp",
],
shared_libs: [
"libfmq",
"libnativewindow",
],
static_libs: [
"android.hardware.neuralnetworks@1.0",
"android.hardware.neuralnetworks@1.1",
"android.hardware.neuralnetworks@1.2",
"android.hardware.neuralnetworks@1.3",
"android.hidl.allocator@1.0",
"android.hidl.memory@1.0",
"libgmock",
"libhidlmemory",
"libneuralnetworks_generated_test_harness",
"libneuralnetworks_utils",
"VtsHalNeuralNetworksV1_0_utils",
"VtsHalNeuralNetworksV1_2Callbacks",
],
whole_static_libs: [
"neuralnetworks_generated_V1_0_example",
"neuralnetworks_generated_V1_1_example",
"neuralnetworks_generated_V1_2_example",
"neuralnetworks_generated_V1_3_example",
],
header_libs: [
"libneuralnetworks_headers",
],
test_suites: ["general-tests"],
}

64
neuralnetworks/1.3/vts/functional/BasicTests.cpp
View file

@ -1,64 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "VtsHalNeuralnetworks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using V1_0::DeviceStatus;
using V1_0::ErrorStatus;
using V1_0::PerformanceInfo;
using V1_2::Constant;
using V1_2::DeviceType;
using V1_2::Extension;
// create device test
TEST_P(NeuralnetworksHidlTest, CreateDevice) {}
// status test
TEST_P(NeuralnetworksHidlTest, StatusTest) {
Return<DeviceStatus> status = kDevice->getStatus();
ASSERT_TRUE(status.isOk());
EXPECT_EQ(DeviceStatus::AVAILABLE, static_cast<DeviceStatus>(status));
}
// initialization
TEST_P(NeuralnetworksHidlTest, GetCapabilitiesTest) {
using OperandPerformance = Capabilities::OperandPerformance;
Return<void> ret = kDevice->getCapabilities_1_3([](ErrorStatus status,
const Capabilities& capabilities) {
EXPECT_EQ(ErrorStatus::NONE, status);
auto isPositive = [](const PerformanceInfo& perf) {
return perf.execTime > 0.0f && perf.powerUsage > 0.0f;
};
EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceScalar));
EXPECT_TRUE(isPositive(capabilities.relaxedFloat32toFloat16PerformanceTensor));
const auto& opPerf = capabilities.operandPerformance;
EXPECT_TRUE(std::all_of(
opPerf.begin(), opPerf.end(),
[isPositive](const OperandPerformance& a) { return isPositive(a.info); }));
EXPECT_TRUE(std::is_sorted(opPerf.begin(), opPerf.end(),
[](const OperandPerformance& a, const OperandPerformance& b) {
return a.type < b.type;
}));
});
EXPECT_TRUE(ret.isOk());
}
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

1377
neuralnetworks/1.3/vts/functional/CompilationCachingTests.cpp
View file

File diff suppressed because it is too large Load diff

418
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
View file

@ -1,418 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "GeneratedTestHarness.h"
#include <android-base/logging.h>
#include <android/hardware/neuralnetworks/1.0/IDevice.h>
#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h>
#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h>
#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h>
#include <android/hardware/neuralnetworks/1.0/types.h>
#include <android/hardware/neuralnetworks/1.1/IDevice.h>
#include <android/hardware/neuralnetworks/1.2/IDevice.h>
#include <android/hardware/neuralnetworks/1.2/IExecutionCallback.h>
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
#include <android/hardware/neuralnetworks/1.2/IPreparedModelCallback.h>
#include <android/hardware/neuralnetworks/1.2/types.h>
#include <android/hardware/neuralnetworks/1.3/IDevice.h>
#include <android/hardware/neuralnetworks/1.3/types.h>
#include <android/hidl/allocator/1.0/IAllocator.h>
#include <android/hidl/memory/1.0/IMemory.h>
#include <hidlmemory/mapping.h>
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <numeric>
#include "1.0/Utils.h"
#include "1.2/Callbacks.h"
#include "ExecutionBurstController.h"
#include "MemoryUtils.h"
#include "TestHarness.h"
#include "Utils.h"
#include "VtsHalNeuralnetworks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using namespace test_helper;
using hidl::memory::V1_0::IMemory;
using V1_0::DataLocation;
using V1_0::ErrorStatus;
using V1_0::OperandLifeTime;
using V1_0::Request;
using V1_1::ExecutionPreference;
using V1_2::Constant;
using V1_2::IPreparedModel;
using V1_2::MeasureTiming;
using V1_2::OperationType;
using V1_2::OutputShape;
using V1_2::SymmPerChannelQuantParams;
using V1_2::Timing;
using V1_2::implementation::ExecutionCallback;
using V1_2::implementation::PreparedModelCallback;
using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
Model createModel(const TestModel& testModel) {
// Model operands.
hidl_vec<Operand> operands(testModel.operands.size());
size_t constCopySize = 0, constRefSize = 0;
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
const auto& op = testModel.operands[i];
DataLocation loc = {};
if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
loc = {.poolIndex = 0,
.offset = static_cast<uint32_t>(constCopySize),
.length = static_cast<uint32_t>(op.data.size())};
constCopySize += op.data.alignedSize();
} else if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
loc = {.poolIndex = 0,
.offset = static_cast<uint32_t>(constRefSize),
.length = static_cast<uint32_t>(op.data.size())};
constRefSize += op.data.alignedSize();
}
Operand::ExtraParams extraParams;
if (op.type == TestOperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL) {
extraParams.channelQuant(SymmPerChannelQuantParams{
.scales = op.channelQuant.scales, .channelDim = op.channelQuant.channelDim});
}
operands[i] = {.type = static_cast<OperandType>(op.type),
.dimensions = op.dimensions,
.numberOfConsumers = op.numberOfConsumers,
.scale = op.scale,
.zeroPoint = op.zeroPoint,
.lifetime = static_cast<OperandLifeTime>(op.lifetime),
.location = loc,
.extraParams = std::move(extraParams)};
}
// Model operations.
hidl_vec<Operation> operations(testModel.operations.size());
std::transform(testModel.operations.begin(), testModel.operations.end(), operations.begin(),
[](const TestOperation& op) -> Operation {
return {.type = static_cast<OperationType>(op.type),
.inputs = op.inputs,
.outputs = op.outputs};
});
// Constant copies.
hidl_vec<uint8_t> operandValues(constCopySize);
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
const auto& op = testModel.operands[i];
if (op.lifetime == TestOperandLifeTime::CONSTANT_COPY) {
const uint8_t* begin = op.data.get<uint8_t>();
const uint8_t* end = begin + op.data.size();
std::copy(begin, end, operandValues.data() + operands[i].location.offset);
}
}
// Shared memory.
hidl_vec<hidl_memory> pools = {};
if (constRefSize > 0) {
hidl_vec_push_back(&pools, nn::allocateSharedMemory(constRefSize));
CHECK_NE(pools[0].size(), 0u);
// load data
sp<IMemory> mappedMemory = mapMemory(pools[0]);
CHECK(mappedMemory.get() != nullptr);
uint8_t* mappedPtr =
reinterpret_cast<uint8_t*>(static_cast<void*>(mappedMemory->getPointer()));
CHECK(mappedPtr != nullptr);
for (uint32_t i = 0; i < testModel.operands.size(); i++) {
const auto& op = testModel.operands[i];
if (op.lifetime == TestOperandLifeTime::CONSTANT_REFERENCE) {
const uint8_t* begin = op.data.get<uint8_t>();
const uint8_t* end = begin + op.data.size();
std::copy(begin, end, mappedPtr + operands[i].location.offset);
}
}
}
return {.operands = std::move(operands),
.operations = std::move(operations),
.inputIndexes = testModel.inputIndexes,
.outputIndexes = testModel.outputIndexes,
.operandValues = std::move(operandValues),
.pools = std::move(pools),
.relaxComputationFloat32toFloat16 = testModel.isRelaxed};
}
static bool isOutputSizeGreaterThanOne(const TestModel& testModel, uint32_t index) {
const auto byteSize = testModel.operands[testModel.outputIndexes[index]].data.size();
return byteSize > 1u;
}
static void makeOutputInsufficientSize(uint32_t outputIndex, Request* request) {
auto& length = request->outputs[outputIndex].location.length;
ASSERT_GT(length, 1u);
length -= 1u;
}
static void makeOutputDimensionsUnspecified(Model* model) {
for (auto i : model->outputIndexes) {
auto& dims = model->operands[i].dimensions;
std::fill(dims.begin(), dims.end(), 0);
}
}
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
sp<ExecutionCallback>& callback) {
return preparedModel->execute_1_2(request, measure, callback);
}
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
hidl_vec<OutputShape>* outputShapes,
Timing* timing) {
ErrorStatus result;
Return<void> ret = preparedModel->executeSynchronously(
request, measure,
[&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
const Timing& time) {
result = error;
*outputShapes = shapes;
*timing = time;
});
if (!ret.isOk()) {
return ErrorStatus::GENERAL_FAILURE;
}
return result;
}
static std::shared_ptr<::android::nn::ExecutionBurstController> CreateBurst(
const sp<IPreparedModel>& preparedModel) {
return android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true);
}
enum class Executor { ASYNC, SYNC, BURST };
void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
Executor executor, MeasureTiming measure, OutputType outputType) {
// If output0 does not have size larger than one byte, we can not test with insufficient buffer.
if (outputType == OutputType::INSUFFICIENT && !isOutputSizeGreaterThanOne(testModel, 0)) {
return;
}
Request request = createRequest(testModel);
if (outputType == OutputType::INSUFFICIENT) {
makeOutputInsufficientSize(/*outputIndex=*/0, &request);
}
ErrorStatus executionStatus;
hidl_vec<OutputShape> outputShapes;
Timing timing;
switch (executor) {
case Executor::ASYNC: {
SCOPED_TRACE("asynchronous");
// launch execution
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
Return<ErrorStatus> executionLaunchStatus =
ExecutePreparedModel(preparedModel, request, measure, executionCallback);
ASSERT_TRUE(executionLaunchStatus.isOk());
EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
// retrieve execution status
executionCallback->wait();
executionStatus = executionCallback->getStatus();
outputShapes = executionCallback->getOutputShapes();
timing = executionCallback->getTiming();
break;
}
case Executor::SYNC: {
SCOPED_TRACE("synchronous");
// execute
Return<ErrorStatus> executionReturnStatus =
ExecutePreparedModel(preparedModel, request, measure, &outputShapes, &timing);
ASSERT_TRUE(executionReturnStatus.isOk());
executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
break;
}
case Executor::BURST: {
SCOPED_TRACE("burst");
// create burst
const std::shared_ptr<::android::nn::ExecutionBurstController> controller =
CreateBurst(preparedModel);
ASSERT_NE(nullptr, controller.get());
// create memory keys
std::vector<intptr_t> keys(request.pools.size());
for (size_t i = 0; i < keys.size(); ++i) {
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
}
// execute burst
std::tie(executionStatus, outputShapes, timing) =
controller->compute(request, measure, keys);
break;
}
}
if (outputType != OutputType::FULLY_SPECIFIED &&
executionStatus == ErrorStatus::GENERAL_FAILURE) {
LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot "
"execute model that it does not support.";
std::cout << "[ ] Early termination of test because vendor service cannot "
"execute model that it does not support."
<< std::endl;
GTEST_SKIP();
}
if (measure == MeasureTiming::NO) {
EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
EXPECT_EQ(UINT64_MAX, timing.timeInDriver);
} else {
if (timing.timeOnDevice != UINT64_MAX && timing.timeInDriver != UINT64_MAX) {
EXPECT_LE(timing.timeOnDevice, timing.timeInDriver);
}
}
switch (outputType) {
case OutputType::FULLY_SPECIFIED:
// If the model output operands are fully specified, outputShapes must be either
// either empty, or have the same number of elements as the number of outputs.
ASSERT_EQ(ErrorStatus::NONE, executionStatus);
ASSERT_TRUE(outputShapes.size() == 0 ||
outputShapes.size() == testModel.outputIndexes.size());
break;
case OutputType::UNSPECIFIED:
// If the model output operands are not fully specified, outputShapes must have
// the same number of elements as the number of outputs.
ASSERT_EQ(ErrorStatus::NONE, executionStatus);
ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
break;
case OutputType::INSUFFICIENT:
ASSERT_EQ(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, executionStatus);
ASSERT_EQ(outputShapes.size(), testModel.outputIndexes.size());
ASSERT_FALSE(outputShapes[0].isSufficient);
return;
}
// Go through all outputs, check returned output shapes.
for (uint32_t i = 0; i < outputShapes.size(); i++) {
EXPECT_TRUE(outputShapes[i].isSufficient);
const auto& expect = testModel.operands[testModel.outputIndexes[i]].dimensions;
const std::vector<uint32_t> actual = outputShapes[i].dimensions;
EXPECT_EQ(expect, actual);
}
// Retrieve execution results.
const std::vector<TestBuffer> outputs = getOutputBuffers(request);
// We want "close-enough" results.
checkResults(testModel, outputs);
}
void EvaluatePreparedModel(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
bool testDynamicOutputShape) {
if (testDynamicOutputShape) {
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
OutputType::UNSPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
OutputType::INSUFFICIENT);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
OutputType::INSUFFICIENT);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
OutputType::INSUFFICIENT);
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
OutputType::INSUFFICIENT);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
OutputType::INSUFFICIENT);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
OutputType::INSUFFICIENT);
} else {
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::NO,
OutputType::FULLY_SPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::NO,
OutputType::FULLY_SPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::NO,
OutputType::FULLY_SPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::ASYNC, MeasureTiming::YES,
OutputType::FULLY_SPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::SYNC, MeasureTiming::YES,
OutputType::FULLY_SPECIFIED);
EvaluatePreparedModel(preparedModel, testModel, Executor::BURST, MeasureTiming::YES,
OutputType::FULLY_SPECIFIED);
}
}
void Execute(const sp<IDevice>& device, const TestModel& testModel, bool testDynamicOutputShape) {
Model model = createModel(testModel);
if (testDynamicOutputShape) {
makeOutputDimensionsUnspecified(&model);
}
sp<IPreparedModel> preparedModel;
createPreparedModel(device, model, &preparedModel);
if (preparedModel == nullptr) return;
EvaluatePreparedModel(preparedModel, testModel, testDynamicOutputShape);
}
void GeneratedTestBase::SetUp() {
testing::TestWithParam<GeneratedTestParam>::SetUp();
ASSERT_NE(kDevice, nullptr);
}
std::vector<NamedModel> getNamedModels(const FilterFn& filter) {
return TestModelManager::get().getTestModels(filter);
}
std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info) {
const auto& [namedDevice, namedModel] = info.param;
return gtestCompliantName(getName(namedDevice) + "_" + getName(namedModel));
}
// Tag for the generated tests
class GeneratedTest : public GeneratedTestBase {};
// Tag for the dynamic output shape tests
class DynamicOutputShapeTest : public GeneratedTest {};
TEST_P(GeneratedTest, Test) {
Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/false);
}
TEST_P(DynamicOutputShapeTest, Test) {
Execute(kDevice, kTestModel, /*testDynamicOutputShape=*/true);
}
INSTANTIATE_GENERATED_TEST(GeneratedTest,
[](const TestModel& testModel) { return !testModel.expectFailure; });
INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest,
[](const TestModel& testModel) { return !testModel.expectFailure; });
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

66
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h
View file

@ -1,66 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
#include <android/hardware/neuralnetworks/1.3/IDevice.h>
#include <android/hardware/neuralnetworks/1.3/types.h>
#include <functional>
#include <vector>
#include "1.0/Utils.h"
#include "TestHarness.h"
#include "VtsHalNeuralnetworks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using NamedModel = Named<const test_helper::TestModel*>;
using GeneratedTestParam = std::tuple<NamedDevice, NamedModel>;
class GeneratedTestBase : public testing::TestWithParam<GeneratedTestParam> {
protected:
void SetUp() override;
const sp<IDevice> kDevice = getData(std::get<NamedDevice>(GetParam()));
const test_helper::TestModel& kTestModel = *getData(std::get<NamedModel>(GetParam()));
};
using FilterFn = std::function<bool(const test_helper::TestModel&)>;
std::vector<NamedModel> getNamedModels(const FilterFn& filter);
std::string printGeneratedTest(const testing::TestParamInfo<GeneratedTestParam>& info);
#define INSTANTIATE_GENERATED_TEST(TestSuite, filter) \
INSTANTIATE_TEST_SUITE_P(TestGenerated, TestSuite, \
testing::Combine(testing::ValuesIn(getNamedDevices()), \
testing::ValuesIn(getNamedModels(filter))), \
printGeneratedTest)
// Tag for the validation tests, instantiated in VtsHalNeuralnetworks.cpp.
// TODO: Clean up the hierarchy for ValidationTest.
class ValidationTest : public GeneratedTestBase {};
Model createModel(const test_helper::TestModel& testModel);
void PrepareModel(const sp<IDevice>& device, const Model& model,
sp<V1_2::IPreparedModel>* preparedModel);
void EvaluatePreparedModel(const sp<V1_2::IPreparedModel>& preparedModel,
const test_helper::TestModel& testModel, bool testDynamicOutputShape);
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional
#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_3_GENERATED_TEST_HARNESS_H

144
neuralnetworks/1.3/vts/functional/TestAssertions.cpp
View file

@ -1,144 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <android/hardware/neuralnetworks/1.3/types.h>
#include "TestHarness.h"
namespace android::hardware::neuralnetworks::V1_3 {
// Make sure that the HIDL enums are compatible with the values defined in
// frameworks/ml/nn/tools/test_generator/test_harness/include/TestHarness.h.
using namespace test_helper;
#define CHECK_TEST_ENUM(EnumType, enumValue) \
static_assert(static_cast<EnumType>(Test##EnumType::enumValue) == EnumType::enumValue)
using V1_2::OperationType;
CHECK_TEST_ENUM(OperandType, FLOAT32);
CHECK_TEST_ENUM(OperandType, INT32);
CHECK_TEST_ENUM(OperandType, UINT32);
CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT32);
CHECK_TEST_ENUM(OperandType, TENSOR_INT32);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM);
CHECK_TEST_ENUM(OperandType, BOOL);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_SYMM);
CHECK_TEST_ENUM(OperandType, TENSOR_FLOAT16);
CHECK_TEST_ENUM(OperandType, TENSOR_BOOL8);
CHECK_TEST_ENUM(OperandType, FLOAT16);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM_PER_CHANNEL);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT16_ASYMM);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_SYMM);
CHECK_TEST_ENUM(OperandType, TENSOR_QUANT8_ASYMM_SIGNED);
CHECK_TEST_ENUM(OperationType, ADD);
CHECK_TEST_ENUM(OperationType, AVERAGE_POOL_2D);
CHECK_TEST_ENUM(OperationType, CONCATENATION);
CHECK_TEST_ENUM(OperationType, CONV_2D);
CHECK_TEST_ENUM(OperationType, DEPTHWISE_CONV_2D);
CHECK_TEST_ENUM(OperationType, DEPTH_TO_SPACE);
CHECK_TEST_ENUM(OperationType, DEQUANTIZE);
CHECK_TEST_ENUM(OperationType, EMBEDDING_LOOKUP);
CHECK_TEST_ENUM(OperationType, FLOOR);
CHECK_TEST_ENUM(OperationType, FULLY_CONNECTED);
CHECK_TEST_ENUM(OperationType, HASHTABLE_LOOKUP);
CHECK_TEST_ENUM(OperationType, L2_NORMALIZATION);
CHECK_TEST_ENUM(OperationType, L2_POOL_2D);
CHECK_TEST_ENUM(OperationType, LOCAL_RESPONSE_NORMALIZATION);
CHECK_TEST_ENUM(OperationType, LOGISTIC);
CHECK_TEST_ENUM(OperationType, LSH_PROJECTION);
CHECK_TEST_ENUM(OperationType, LSTM);
CHECK_TEST_ENUM(OperationType, MAX_POOL_2D);
CHECK_TEST_ENUM(OperationType, MUL);
CHECK_TEST_ENUM(OperationType, RELU);
CHECK_TEST_ENUM(OperationType, RELU1);
CHECK_TEST_ENUM(OperationType, RELU6);
CHECK_TEST_ENUM(OperationType, RESHAPE);
CHECK_TEST_ENUM(OperationType, RESIZE_BILINEAR);
CHECK_TEST_ENUM(OperationType, RNN);
CHECK_TEST_ENUM(OperationType, SOFTMAX);
CHECK_TEST_ENUM(OperationType, SPACE_TO_DEPTH);
CHECK_TEST_ENUM(OperationType, SVDF);
CHECK_TEST_ENUM(OperationType, TANH);
CHECK_TEST_ENUM(OperationType, BATCH_TO_SPACE_ND);
CHECK_TEST_ENUM(OperationType, DIV);
CHECK_TEST_ENUM(OperationType, MEAN);
CHECK_TEST_ENUM(OperationType, PAD);
CHECK_TEST_ENUM(OperationType, SPACE_TO_BATCH_ND);
CHECK_TEST_ENUM(OperationType, SQUEEZE);
CHECK_TEST_ENUM(OperationType, STRIDED_SLICE);
CHECK_TEST_ENUM(OperationType, SUB);
CHECK_TEST_ENUM(OperationType, TRANSPOSE);
CHECK_TEST_ENUM(OperationType, ABS);
CHECK_TEST_ENUM(OperationType, ARGMAX);
CHECK_TEST_ENUM(OperationType, ARGMIN);
CHECK_TEST_ENUM(OperationType, AXIS_ALIGNED_BBOX_TRANSFORM);
CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_LSTM);
CHECK_TEST_ENUM(OperationType, BIDIRECTIONAL_SEQUENCE_RNN);
CHECK_TEST_ENUM(OperationType, BOX_WITH_NMS_LIMIT);
CHECK_TEST_ENUM(OperationType, CAST);
CHECK_TEST_ENUM(OperationType, CHANNEL_SHUFFLE);
CHECK_TEST_ENUM(OperationType, DETECTION_POSTPROCESSING);
CHECK_TEST_ENUM(OperationType, EQUAL);
CHECK_TEST_ENUM(OperationType, EXP);
CHECK_TEST_ENUM(OperationType, EXPAND_DIMS);
CHECK_TEST_ENUM(OperationType, GATHER);
CHECK_TEST_ENUM(OperationType, GENERATE_PROPOSALS);
CHECK_TEST_ENUM(OperationType, GREATER);
CHECK_TEST_ENUM(OperationType, GREATER_EQUAL);
CHECK_TEST_ENUM(OperationType, GROUPED_CONV_2D);
CHECK_TEST_ENUM(OperationType, HEATMAP_MAX_KEYPOINT);
CHECK_TEST_ENUM(OperationType, INSTANCE_NORMALIZATION);
CHECK_TEST_ENUM(OperationType, LESS);
CHECK_TEST_ENUM(OperationType, LESS_EQUAL);
CHECK_TEST_ENUM(OperationType, LOG);
CHECK_TEST_ENUM(OperationType, LOGICAL_AND);
CHECK_TEST_ENUM(OperationType, LOGICAL_NOT);
CHECK_TEST_ENUM(OperationType, LOGICAL_OR);
CHECK_TEST_ENUM(OperationType, LOG_SOFTMAX);
CHECK_TEST_ENUM(OperationType, MAXIMUM);
CHECK_TEST_ENUM(OperationType, MINIMUM);
CHECK_TEST_ENUM(OperationType, NEG);
CHECK_TEST_ENUM(OperationType, NOT_EQUAL);
CHECK_TEST_ENUM(OperationType, PAD_V2);
CHECK_TEST_ENUM(OperationType, POW);
CHECK_TEST_ENUM(OperationType, PRELU);
CHECK_TEST_ENUM(OperationType, QUANTIZE);
CHECK_TEST_ENUM(OperationType, QUANTIZED_16BIT_LSTM);
CHECK_TEST_ENUM(OperationType, RANDOM_MULTINOMIAL);
CHECK_TEST_ENUM(OperationType, REDUCE_ALL);
CHECK_TEST_ENUM(OperationType, REDUCE_ANY);
CHECK_TEST_ENUM(OperationType, REDUCE_MAX);
CHECK_TEST_ENUM(OperationType, REDUCE_MIN);
CHECK_TEST_ENUM(OperationType, REDUCE_PROD);
CHECK_TEST_ENUM(OperationType, REDUCE_SUM);
CHECK_TEST_ENUM(OperationType, ROI_ALIGN);
CHECK_TEST_ENUM(OperationType, ROI_POOLING);
CHECK_TEST_ENUM(OperationType, RSQRT);
CHECK_TEST_ENUM(OperationType, SELECT);
CHECK_TEST_ENUM(OperationType, SIN);
CHECK_TEST_ENUM(OperationType, SLICE);
CHECK_TEST_ENUM(OperationType, SPLIT);
CHECK_TEST_ENUM(OperationType, SQRT);
CHECK_TEST_ENUM(OperationType, TILE);
CHECK_TEST_ENUM(OperationType, TOPK_V2);
CHECK_TEST_ENUM(OperationType, TRANSPOSE_CONV_2D);
CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_LSTM);
CHECK_TEST_ENUM(OperationType, UNIDIRECTIONAL_SEQUENCE_RNN);
CHECK_TEST_ENUM(OperationType, RESIZE_NEAREST_NEIGHBOR);
#undef CHECK_TEST_ENUM
} // namespace android::hardware::neuralnetworks::V1_3

407
neuralnetworks/1.3/vts/functional/ValidateBurst.cpp
View file

@ -1,407 +0,0 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "VtsHalNeuralnetworks.h"
#include "1.2/Callbacks.h"
#include "ExecutionBurstController.h"
#include "ExecutionBurstServer.h"
#include "GeneratedTestHarness.h"
#include "TestHarness.h"
#include "Utils.h"
#include <android-base/logging.h>
#include <cstring>
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using nn::ExecutionBurstController;
using nn::RequestChannelSender;
using nn::ResultChannelReceiver;
using V1_0::ErrorStatus;
using V1_0::Request;
using V1_2::FmqRequestDatum;
using V1_2::FmqResultDatum;
using V1_2::IBurstCallback;
using V1_2::IBurstContext;
using V1_2::IPreparedModel;
using V1_2::MeasureTiming;
using V1_2::Timing;
using ExecutionBurstCallback = ExecutionBurstController::ExecutionBurstCallback;
// This constant value represents the length of an FMQ that is large enough to
// return a result from a burst execution for all of the generated test cases.
constexpr size_t kExecutionBurstChannelLength = 1024;
// This constant value represents a length of an FMQ that is not large enough
// to return a result from a burst execution for some of the generated test
// cases.
constexpr size_t kExecutionBurstChannelSmallLength = 8;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
static bool badTiming(Timing timing) {
return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
}
static void createBurst(const sp<IPreparedModel>& preparedModel, const sp<IBurstCallback>& callback,
std::unique_ptr<RequestChannelSender>* sender,
std::unique_ptr<ResultChannelReceiver>* receiver,
sp<IBurstContext>* context,
size_t resultChannelLength = kExecutionBurstChannelLength) {
ASSERT_NE(nullptr, preparedModel.get());
ASSERT_NE(nullptr, sender);
ASSERT_NE(nullptr, receiver);
ASSERT_NE(nullptr, context);
// create FMQ objects
auto [fmqRequestChannel, fmqRequestDescriptor] =
RequestChannelSender::create(kExecutionBurstChannelLength, /*blocking=*/true);
auto [fmqResultChannel, fmqResultDescriptor] =
ResultChannelReceiver::create(resultChannelLength, /*blocking=*/true);
ASSERT_NE(nullptr, fmqRequestChannel.get());
ASSERT_NE(nullptr, fmqResultChannel.get());
ASSERT_NE(nullptr, fmqRequestDescriptor);
ASSERT_NE(nullptr, fmqResultDescriptor);
// configure burst
ErrorStatus errorStatus;
sp<IBurstContext> burstContext;
const Return<void> ret = preparedModel->configureExecutionBurst(
callback, *fmqRequestDescriptor, *fmqResultDescriptor,
[&errorStatus, &burstContext](ErrorStatus status, const sp<IBurstContext>& context) {
errorStatus = status;
burstContext = context;
});
ASSERT_TRUE(ret.isOk());
ASSERT_EQ(ErrorStatus::NONE, errorStatus);
ASSERT_NE(nullptr, burstContext.get());
// return values
*sender = std::move(fmqRequestChannel);
*receiver = std::move(fmqResultChannel);
*context = burstContext;
}
static void createBurstWithResultChannelLength(
const sp<IPreparedModel>& preparedModel, size_t resultChannelLength,
std::shared_ptr<ExecutionBurstController>* controller) {
ASSERT_NE(nullptr, preparedModel.get());
ASSERT_NE(nullptr, controller);
// create FMQ objects
std::unique_ptr<RequestChannelSender> sender;
std::unique_ptr<ResultChannelReceiver> receiver;
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
sp<IBurstContext> context;
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context,
resultChannelLength));
ASSERT_NE(nullptr, sender.get());
ASSERT_NE(nullptr, receiver.get());
ASSERT_NE(nullptr, context.get());
// return values
*controller = std::make_shared<ExecutionBurstController>(std::move(sender), std::move(receiver),
context, callback);
}
// Primary validation function. This function will take a valid serialized
// request, apply a mutation to it to invalidate the serialized request, then
// pass it to interface calls that use the serialized request. Note that the
// serialized request here is passed by value, and any mutation to the
// serialized request does not leave this function.
static void validate(RequestChannelSender* sender, ResultChannelReceiver* receiver,
const std::string& message, std::vector<FmqRequestDatum> serialized,
const std::function<void(std::vector<FmqRequestDatum>*)>& mutation) {
mutation(&serialized);
// skip if packet is too large to send
if (serialized.size() > kExecutionBurstChannelLength) {
return;
}
SCOPED_TRACE(message);
// send invalid packet
ASSERT_TRUE(sender->sendPacket(serialized));
// receive error
auto results = receiver->getBlocking();
ASSERT_TRUE(results.has_value());
const auto [status, outputShapes, timing] = std::move(*results);
EXPECT_NE(ErrorStatus::NONE, status);
EXPECT_EQ(0u, outputShapes.size());
EXPECT_TRUE(badTiming(timing));
}
// For validation, valid packet entries are mutated to invalid packet entries,
// or invalid packet entries are inserted into valid packets. This function
// creates pre-set invalid packet entries for convenience.
static std::vector<FmqRequestDatum> createBadRequestPacketEntries() {
const FmqRequestDatum::PacketInformation packetInformation = {
/*.packetSize=*/10, /*.numberOfInputOperands=*/10, /*.numberOfOutputOperands=*/10,
/*.numberOfPools=*/10};
const FmqRequestDatum::OperandInformation operandInformation = {
/*.hasNoValue=*/false, /*.location=*/{}, /*.numberOfDimensions=*/10};
const int32_t invalidPoolIdentifier = std::numeric_limits<int32_t>::max();
std::vector<FmqRequestDatum> bad(7);
bad[0].packetInformation(packetInformation);
bad[1].inputOperandInformation(operandInformation);
bad[2].inputOperandDimensionValue(0);
bad[3].outputOperandInformation(operandInformation);
bad[4].outputOperandDimensionValue(0);
bad[5].poolIdentifier(invalidPoolIdentifier);
bad[6].measureTiming(MeasureTiming::YES);
return bad;
}
// For validation, valid packet entries are mutated to invalid packet entries,
// or invalid packet entries are inserted into valid packets. This function
// retrieves pre-set invalid packet entries for convenience. This function
// caches these data so they can be reused on subsequent validation checks.
static const std::vector<FmqRequestDatum>& getBadRequestPacketEntries() {
static const std::vector<FmqRequestDatum> bad = createBadRequestPacketEntries();
return bad;
}
///////////////////////// REMOVE DATUM ////////////////////////////////////
static void removeDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
const std::vector<FmqRequestDatum>& serialized) {
for (size_t index = 0; index < serialized.size(); ++index) {
const std::string message = "removeDatum: removed datum at index " + std::to_string(index);
validate(sender, receiver, message, serialized,
[index](std::vector<FmqRequestDatum>* serialized) {
serialized->erase(serialized->begin() + index);
});
}
}
///////////////////////// ADD DATUM ////////////////////////////////////
static void addDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
const std::vector<FmqRequestDatum>& serialized) {
const std::vector<FmqRequestDatum>& extra = getBadRequestPacketEntries();
for (size_t index = 0; index <= serialized.size(); ++index) {
for (size_t type = 0; type < extra.size(); ++type) {
const std::string message = "addDatum: added datum type " + std::to_string(type) +
" at index " + std::to_string(index);
validate(sender, receiver, message, serialized,
[index, type, &extra](std::vector<FmqRequestDatum>* serialized) {
serialized->insert(serialized->begin() + index, extra[type]);
});
}
}
}
///////////////////////// MUTATE DATUM ////////////////////////////////////
static bool interestingCase(const FmqRequestDatum& lhs, const FmqRequestDatum& rhs) {
using Discriminator = FmqRequestDatum::hidl_discriminator;
const bool differentValues = (lhs != rhs);
const bool sameDiscriminator = (lhs.getDiscriminator() == rhs.getDiscriminator());
const auto discriminator = rhs.getDiscriminator();
const bool isDimensionValue = (discriminator == Discriminator::inputOperandDimensionValue ||
discriminator == Discriminator::outputOperandDimensionValue);
return differentValues && !(sameDiscriminator && isDimensionValue);
}
static void mutateDatumTest(RequestChannelSender* sender, ResultChannelReceiver* receiver,
const std::vector<FmqRequestDatum>& serialized) {
const std::vector<FmqRequestDatum>& change = getBadRequestPacketEntries();
for (size_t index = 0; index < serialized.size(); ++index) {
for (size_t type = 0; type < change.size(); ++type) {
if (interestingCase(serialized[index], change[type])) {
const std::string message = "mutateDatum: changed datum at index " +
std::to_string(index) + " to datum type " +
std::to_string(type);
validate(sender, receiver, message, serialized,
[index, type, &change](std::vector<FmqRequestDatum>* serialized) {
(*serialized)[index] = change[type];
});
}
}
}
}
///////////////////////// BURST VALIATION TESTS ////////////////////////////////////
static void validateBurstSerialization(const sp<IPreparedModel>& preparedModel,
const Request& request) {
// create burst
std::unique_ptr<RequestChannelSender> sender;
std::unique_ptr<ResultChannelReceiver> receiver;
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
sp<IBurstContext> context;
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
ASSERT_NE(nullptr, sender.get());
ASSERT_NE(nullptr, receiver.get());
ASSERT_NE(nullptr, context.get());
// load memory into callback slots
std::vector<intptr_t> keys;
keys.reserve(request.pools.size());
std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
[](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
// ensure slot std::numeric_limits<int32_t>::max() doesn't exist (for
// subsequent slot validation testing)
ASSERT_TRUE(std::all_of(slots.begin(), slots.end(), [](int32_t slot) {
return slot != std::numeric_limits<int32_t>::max();
}));
// serialize the request
const auto serialized = android::nn::serialize(request, MeasureTiming::YES, slots);
// validations
removeDatumTest(sender.get(), receiver.get(), serialized);
addDatumTest(sender.get(), receiver.get(), serialized);
mutateDatumTest(sender.get(), receiver.get(), serialized);
}
// This test validates that when the Result message size exceeds length of the
// result FMQ, the service instance gracefully fails and returns an error.
static void validateBurstFmqLength(const sp<IPreparedModel>& preparedModel,
const Request& request) {
// create regular burst
std::shared_ptr<ExecutionBurstController> controllerRegular;
ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
preparedModel, kExecutionBurstChannelLength, &controllerRegular));
ASSERT_NE(nullptr, controllerRegular.get());
// create burst with small output channel
std::shared_ptr<ExecutionBurstController> controllerSmall;
ASSERT_NO_FATAL_FAILURE(createBurstWithResultChannelLength(
preparedModel, kExecutionBurstChannelSmallLength, &controllerSmall));
ASSERT_NE(nullptr, controllerSmall.get());
// load memory into callback slots
std::vector<intptr_t> keys(request.pools.size());
for (size_t i = 0; i < keys.size(); ++i) {
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
}
// collect serialized result by running regular burst
const auto [statusRegular, outputShapesRegular, timingRegular] =
controllerRegular->compute(request, MeasureTiming::NO, keys);
// skip test if regular burst output isn't useful for testing a failure
// caused by having too small of a length for the result FMQ
const std::vector<FmqResultDatum> serialized =
android::nn::serialize(statusRegular, outputShapesRegular, timingRegular);
if (statusRegular != ErrorStatus::NONE ||
serialized.size() <= kExecutionBurstChannelSmallLength) {
return;
}
// by this point, execution should fail because the result channel isn't
// large enough to return the serialized result
const auto [statusSmall, outputShapesSmall, timingSmall] =
controllerSmall->compute(request, MeasureTiming::NO, keys);
EXPECT_NE(ErrorStatus::NONE, statusSmall);
EXPECT_EQ(0u, outputShapesSmall.size());
EXPECT_TRUE(badTiming(timingSmall));
}
static bool isSanitized(const FmqResultDatum& datum) {
using Discriminator = FmqResultDatum::hidl_discriminator;
// check to ensure the padding values in the returned
// FmqResultDatum::OperandInformation are initialized to 0
if (datum.getDiscriminator() == Discriminator::operandInformation) {
static_assert(
offsetof(FmqResultDatum::OperandInformation, isSufficient) == 0,
"unexpected value for offset of FmqResultDatum::OperandInformation::isSufficient");
static_assert(
sizeof(FmqResultDatum::OperandInformation::isSufficient) == 1,
"unexpected value for size of FmqResultDatum::OperandInformation::isSufficient");
static_assert(offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) == 4,
"unexpected value for offset of "
"FmqResultDatum::OperandInformation::numberOfDimensions");
static_assert(sizeof(FmqResultDatum::OperandInformation::numberOfDimensions) == 4,
"unexpected value for size of "
"FmqResultDatum::OperandInformation::numberOfDimensions");
static_assert(sizeof(FmqResultDatum::OperandInformation) == 8,
"unexpected value for size of "
"FmqResultDatum::OperandInformation");
constexpr size_t paddingOffset =
offsetof(FmqResultDatum::OperandInformation, isSufficient) +
sizeof(FmqResultDatum::OperandInformation::isSufficient);
constexpr size_t paddingSize =
offsetof(FmqResultDatum::OperandInformation, numberOfDimensions) - paddingOffset;
FmqResultDatum::OperandInformation initialized{};
std::memset(&initialized, 0, sizeof(initialized));
const char* initializedPaddingStart =
reinterpret_cast<const char*>(&initialized) + paddingOffset;
const char* datumPaddingStart =
reinterpret_cast<const char*>(&datum.operandInformation()) + paddingOffset;
return std::memcmp(datumPaddingStart, initializedPaddingStart, paddingSize) == 0;
}
// there are no other padding initialization checks required, so return true
// for any sum-type that isn't FmqResultDatum::OperandInformation
return true;
}
static void validateBurstSanitized(const sp<IPreparedModel>& preparedModel,
const Request& request) {
// create burst
std::unique_ptr<RequestChannelSender> sender;
std::unique_ptr<ResultChannelReceiver> receiver;
sp<ExecutionBurstCallback> callback = new ExecutionBurstCallback();
sp<IBurstContext> context;
ASSERT_NO_FATAL_FAILURE(createBurst(preparedModel, callback, &sender, &receiver, &context));
ASSERT_NE(nullptr, sender.get());
ASSERT_NE(nullptr, receiver.get());
ASSERT_NE(nullptr, context.get());
// load memory into callback slots
std::vector<intptr_t> keys;
keys.reserve(request.pools.size());
std::transform(request.pools.begin(), request.pools.end(), std::back_inserter(keys),
[](const auto& pool) { return reinterpret_cast<intptr_t>(&pool); });
const std::vector<int32_t> slots = callback->getSlots(request.pools, keys);
// send valid request
ASSERT_TRUE(sender->send(request, MeasureTiming::YES, slots));
// receive valid result
auto serialized = receiver->getPacketBlocking();
ASSERT_TRUE(serialized.has_value());
// sanitize result
ASSERT_TRUE(std::all_of(serialized->begin(), serialized->end(), isSanitized))
<< "The result serialized data is not properly sanitized";
}
///////////////////////////// ENTRY POINT //////////////////////////////////
void validateBurst(const sp<IPreparedModel>& preparedModel, const Request& request) {
ASSERT_NO_FATAL_FAILURE(validateBurstSerialization(preparedModel, request));
ASSERT_NO_FATAL_FAILURE(validateBurstFmqLength(preparedModel, request));
ASSERT_NO_FATAL_FAILURE(validateBurstSanitized(preparedModel, request));
}
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

718
neuralnetworks/1.3/vts/functional/ValidateModel.cpp
View file

@ -1,718 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "1.0/Utils.h"
#include "1.2/Callbacks.h"
#include "GeneratedTestHarness.h"
#include "VtsHalNeuralnetworks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using V1_0::ErrorStatus;
using V1_0::OperandLifeTime;
using V1_1::ExecutionPreference;
using V1_2::IPreparedModel;
using V1_2::OperationType;
using V1_2::OperationTypeRange;
using V1_2::SymmPerChannelQuantParams;
using V1_2::implementation::PreparedModelCallback;
using HidlToken =
hidl_array<uint8_t, static_cast<uint32_t>(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
static void validateGetSupportedOperations(const sp<IDevice>& device, const std::string& message,
const Model& model) {
SCOPED_TRACE(message + " [getSupportedOperations_1_3]");
Return<void> ret = device->getSupportedOperations_1_3(
model, [&](ErrorStatus status, const hidl_vec<bool>&) {
EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, status);
});
EXPECT_TRUE(ret.isOk());
}
static void validatePrepareModel(const sp<IDevice>& device, const std::string& message,
const Model& model, ExecutionPreference preference) {
SCOPED_TRACE(message + " [prepareModel_1_3]");
sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
Return<ErrorStatus> prepareLaunchStatus =
device->prepareModel_1_3(model, preference, hidl_vec<hidl_handle>(),
hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
ASSERT_TRUE(prepareLaunchStatus.isOk());
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(prepareLaunchStatus));
preparedModelCallback->wait();
ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, prepareReturnStatus);
sp<IPreparedModel> preparedModel = getPreparedModel_1_2(preparedModelCallback);
ASSERT_EQ(nullptr, preparedModel.get());
}
static bool validExecutionPreference(ExecutionPreference preference) {
return preference == ExecutionPreference::LOW_POWER ||
preference == ExecutionPreference::FAST_SINGLE_ANSWER ||
preference == ExecutionPreference::SUSTAINED_SPEED;
}
// Primary validation function. This function will take a valid model, apply a
// mutation to it to invalidate the model, then pass it to interface calls that
// use the model. Note that the model here is passed by value, and any mutation
// to the model does not leave this function.
static void validate(const sp<IDevice>& device, const std::string& message, Model model,
const std::function<void(Model*)>& mutation,
ExecutionPreference preference = ExecutionPreference::FAST_SINGLE_ANSWER) {
mutation(&model);
if (validExecutionPreference(preference)) {
validateGetSupportedOperations(device, message, model);
}
validatePrepareModel(device, message, model, preference);
}
static uint32_t addOperand(Model* model) {
return hidl_vec_push_back(&model->operands,
{
.type = OperandType::INT32,
.dimensions = {},
.numberOfConsumers = 0,
.scale = 0.0f,
.zeroPoint = 0,
.lifetime = OperandLifeTime::MODEL_INPUT,
.location = {.poolIndex = 0, .offset = 0, .length = 0},
});
}
static uint32_t addOperand(Model* model, OperandLifeTime lifetime) {
uint32_t index = addOperand(model);
model->operands[index].numberOfConsumers = 1;
model->operands[index].lifetime = lifetime;
return index;
}
///////////////////////// VALIDATE MODEL OPERAND TYPE /////////////////////////
static const uint32_t invalidOperandTypes[] = {
static_cast<uint32_t>(OperandTypeRange::FUNDAMENTAL_MIN) - 1,
static_cast<uint32_t>(OperandTypeRange::FUNDAMENTAL_MAX) + 1,
static_cast<uint32_t>(OperandTypeRange::OEM_MIN) - 1,
static_cast<uint32_t>(OperandTypeRange::OEM_MAX) + 1,
};
static void mutateOperandTypeTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
for (uint32_t invalidOperandType : invalidOperandTypes) {
const std::string message = "mutateOperandTypeTest: operand " +
std::to_string(operand) + " set to value " +
std::to_string(invalidOperandType);
validate(device, message, model, [operand, invalidOperandType](Model* model) {
model->operands[operand].type = static_cast<OperandType>(invalidOperandType);
});
}
}
}
///////////////////////// VALIDATE OPERAND RANK /////////////////////////
static uint32_t getInvalidRank(OperandType type) {
switch (type) {
case OperandType::FLOAT16:
case OperandType::FLOAT32:
case OperandType::INT32:
case OperandType::UINT32:
case OperandType::BOOL:
return 1;
case OperandType::TENSOR_BOOL8:
case OperandType::TENSOR_FLOAT16:
case OperandType::TENSOR_FLOAT32:
case OperandType::TENSOR_INT32:
case OperandType::TENSOR_QUANT8_ASYMM:
case OperandType::TENSOR_QUANT8_SYMM:
case OperandType::TENSOR_QUANT16_ASYMM:
case OperandType::TENSOR_QUANT16_SYMM:
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
return 0;
default:
return 0;
}
}
static void mutateOperandRankTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
const uint32_t invalidRank = getInvalidRank(model.operands[operand].type);
if (invalidRank == 0) {
continue;
}
const std::string message = "mutateOperandRankTest: operand " + std::to_string(operand) +
" has rank of " + std::to_string(invalidRank);
validate(device, message, model, [operand, invalidRank](Model* model) {
model->operands[operand].dimensions = std::vector<uint32_t>(invalidRank, 0);
});
}
}
///////////////////////// VALIDATE OPERAND SCALE /////////////////////////
static float getInvalidScale(OperandType type) {
switch (type) {
case OperandType::FLOAT16:
case OperandType::FLOAT32:
case OperandType::INT32:
case OperandType::UINT32:
case OperandType::BOOL:
case OperandType::TENSOR_BOOL8:
case OperandType::TENSOR_FLOAT16:
case OperandType::TENSOR_FLOAT32:
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
return 1.0f;
case OperandType::TENSOR_INT32:
return -1.0f;
case OperandType::TENSOR_QUANT8_SYMM:
case OperandType::TENSOR_QUANT8_ASYMM:
case OperandType::TENSOR_QUANT16_ASYMM:
case OperandType::TENSOR_QUANT16_SYMM:
return 0.0f;
default:
return 0.0f;
}
}
static void mutateOperandScaleTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
const float invalidScale = getInvalidScale(model.operands[operand].type);
const std::string message = "mutateOperandScaleTest: operand " + std::to_string(operand) +
" has scale of " + std::to_string(invalidScale);
validate(device, message, model, [operand, invalidScale](Model* model) {
model->operands[operand].scale = invalidScale;
});
}
}
///////////////////////// VALIDATE OPERAND ZERO POINT /////////////////////////
static std::vector<int32_t> getInvalidZeroPoints(OperandType type) {
switch (type) {
case OperandType::FLOAT16:
case OperandType::FLOAT32:
case OperandType::INT32:
case OperandType::UINT32:
case OperandType::BOOL:
case OperandType::TENSOR_BOOL8:
case OperandType::TENSOR_FLOAT16:
case OperandType::TENSOR_FLOAT32:
case OperandType::TENSOR_INT32:
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
return {1};
case OperandType::TENSOR_QUANT8_ASYMM:
return {-1, 256};
case OperandType::TENSOR_QUANT8_SYMM:
return {-129, -1, 1, 128};
case OperandType::TENSOR_QUANT16_ASYMM:
return {-1, 65536};
case OperandType::TENSOR_QUANT16_SYMM:
return {-32769, -1, 1, 32768};
default:
return {};
}
}
static void mutateOperandZeroPointTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
const std::vector<int32_t> invalidZeroPoints =
getInvalidZeroPoints(model.operands[operand].type);
for (int32_t invalidZeroPoint : invalidZeroPoints) {
const std::string message = "mutateOperandZeroPointTest: operand " +
std::to_string(operand) + " has zero point of " +
std::to_string(invalidZeroPoint);
validate(device, message, model, [operand, invalidZeroPoint](Model* model) {
model->operands[operand].zeroPoint = invalidZeroPoint;
});
}
}
}
///////////////////////// VALIDATE EXTRA ??? /////////////////////////
// TODO: Operand::lifetime
// TODO: Operand::location
///////////////////////// VALIDATE OPERATION OPERAND TYPE /////////////////////////
static void mutateOperand(Operand* operand, OperandType type) {
Operand newOperand = *operand;
newOperand.type = type;
switch (type) {
case OperandType::FLOAT16:
case OperandType::FLOAT32:
case OperandType::INT32:
case OperandType::UINT32:
case OperandType::BOOL:
newOperand.dimensions = hidl_vec<uint32_t>();
newOperand.scale = 0.0f;
newOperand.zeroPoint = 0;
break;
case OperandType::TENSOR_BOOL8:
case OperandType::TENSOR_FLOAT16:
case OperandType::TENSOR_FLOAT32:
newOperand.dimensions =
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
newOperand.scale = 0.0f;
newOperand.zeroPoint = 0;
break;
case OperandType::TENSOR_INT32:
newOperand.dimensions =
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
newOperand.zeroPoint = 0;
break;
case OperandType::TENSOR_QUANT8_ASYMM:
case OperandType::TENSOR_QUANT8_SYMM:
case OperandType::TENSOR_QUANT16_ASYMM:
case OperandType::TENSOR_QUANT16_SYMM:
newOperand.dimensions =
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
newOperand.scale = operand->scale != 0.0f ? operand->scale : 1.0f;
break;
case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL: {
newOperand.dimensions =
operand->dimensions.size() > 0 ? operand->dimensions : hidl_vec<uint32_t>({1});
newOperand.scale = 0.0f;
newOperand.zeroPoint = 0;
SymmPerChannelQuantParams channelQuant;
channelQuant.channelDim = 0;
channelQuant.scales = hidl_vec<float>(
operand->dimensions.size() > 0 ? static_cast<size_t>(operand->dimensions[0])
: 0);
for (size_t i = 0; i < channelQuant.scales.size(); ++i) {
channelQuant.scales[i] = 1.0f;
}
newOperand.extraParams.channelQuant(std::move(channelQuant));
} break;
case OperandType::OEM:
case OperandType::TENSOR_OEM_BYTE:
default:
break;
}
*operand = newOperand;
}
static bool mutateOperationOperandTypeSkip(size_t operand, OperandType type, const Model& model) {
// Do not test OEM types
if (type == model.operands[operand].type || type == OperandType::OEM ||
type == OperandType::TENSOR_OEM_BYTE) {
return true;
}
for (const Operation& operation : model.operations) {
// Skip mutateOperationOperandTypeTest for the following operations.
// - LSH_PROJECTION's second argument is allowed to have any type.
// - ARGMIN and ARGMAX's first argument can be any of
// TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM).
// - CAST's argument can be any of TENSOR_(FLOAT16|FLOAT32|INT32|QUANT8_ASYMM).
// - RANDOM_MULTINOMIAL's argument can be either TENSOR_FLOAT16 or TENSOR_FLOAT32.
// - DEQUANTIZE input can be any of
// TENSOR_(QUANT8_ASYMM|QUANT8_SYMM|QUANT8_SYMM_PER_CHANNEL), output can
// be of either TENSOR_FLOAT16 or TENSOR_FLOAT32.
// - QUANTIZE input can be either TENSOR_FLOAT16 or TENSOR_FLOAT32
// - CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
// - DEPTHWISE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
// - GROUPED_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
// - TRANSPOSE_CONV_2D filter type (arg 1) can be QUANT8_ASYMM or QUANT8_SYMM_PER_CHANNEL
switch (operation.type) {
case OperationType::LSH_PROJECTION: {
if (operand == operation.inputs[1]) {
return true;
}
} break;
case OperationType::CAST:
case OperationType::ARGMAX:
case OperationType::ARGMIN: {
if (type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32 ||
type == OperandType::TENSOR_INT32 || type == OperandType::TENSOR_QUANT8_ASYMM) {
return true;
}
} break;
case OperationType::QUANTIZE:
case OperationType::RANDOM_MULTINOMIAL: {
if (operand == operation.inputs[0] &&
(type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) {
return true;
}
} break;
case OperationType::DEQUANTIZE: {
if (operand == operation.inputs[0] &&
(type == OperandType::TENSOR_QUANT8_ASYMM ||
type == OperandType::TENSOR_QUANT8_SYMM ||
type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) {
return true;
}
if (operand == operation.outputs[0] &&
(type == OperandType::TENSOR_FLOAT16 || type == OperandType::TENSOR_FLOAT32)) {
return true;
}
} break;
case OperationType::TRANSPOSE_CONV_2D:
case OperationType::GROUPED_CONV_2D:
case OperationType::DEPTHWISE_CONV_2D:
case OperationType::CONV_2D: {
if (operand == operation.inputs[1] &&
(type == OperandType::TENSOR_QUANT8_ASYMM ||
type == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL)) {
return true;
}
} break;
default:
break;
}
}
return false;
}
static void mutateOperationOperandTypeTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
for (OperandType invalidOperandType : hidl_enum_range<OperandType>{}) {
if (mutateOperationOperandTypeSkip(operand, invalidOperandType, model)) {
continue;
}
const std::string message = "mutateOperationOperandTypeTest: operand " +
std::to_string(operand) + " set to type " +
toString(invalidOperandType);
validate(device, message, model, [operand, invalidOperandType](Model* model) {
mutateOperand(&model->operands[operand], invalidOperandType);
});
}
}
}
///////////////////////// VALIDATE MODEL OPERATION TYPE /////////////////////////
static const uint32_t invalidOperationTypes[] = {
static_cast<uint32_t>(OperationTypeRange::FUNDAMENTAL_MAX) + 1,
static_cast<uint32_t>(OperationTypeRange::OEM_MIN) - 1,
static_cast<uint32_t>(OperationTypeRange::OEM_MAX) + 1,
};
static void mutateOperationTypeTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
for (uint32_t invalidOperationType : invalidOperationTypes) {
const std::string message = "mutateOperationTypeTest: operation " +
std::to_string(operation) + " set to value " +
std::to_string(invalidOperationType);
validate(device, message, model, [operation, invalidOperationType](Model* model) {
model->operations[operation].type =
static_cast<OperationType>(invalidOperationType);
});
}
}
}
///////////////////////// VALIDATE MODEL OPERATION INPUT OPERAND INDEX /////////////////////////
static void mutateOperationInputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
const uint32_t invalidOperand = model.operands.size();
for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
const std::string message = "mutateOperationInputOperandIndexTest: operation " +
std::to_string(operation) + " input " +
std::to_string(input);
validate(device, message, model, [operation, input, invalidOperand](Model* model) {
model->operations[operation].inputs[input] = invalidOperand;
});
}
}
}
///////////////////////// VALIDATE MODEL OPERATION OUTPUT OPERAND INDEX /////////////////////////
static void mutateOperationOutputOperandIndexTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
const uint32_t invalidOperand = model.operands.size();
for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
const std::string message = "mutateOperationOutputOperandIndexTest: operation " +
std::to_string(operation) + " output " +
std::to_string(output);
validate(device, message, model, [operation, output, invalidOperand](Model* model) {
model->operations[operation].outputs[output] = invalidOperand;
});
}
}
}
///////////////////////// REMOVE OPERAND FROM EVERYTHING /////////////////////////
static void removeValueAndDecrementGreaterValues(hidl_vec<uint32_t>* vec, uint32_t value) {
if (vec) {
// remove elements matching "value"
auto last = std::remove(vec->begin(), vec->end(), value);
vec->resize(std::distance(vec->begin(), last));
// decrement elements exceeding "value"
std::transform(vec->begin(), vec->end(), vec->begin(),
[value](uint32_t v) { return v > value ? v-- : v; });
}
}
static void removeOperand(Model* model, uint32_t index) {
hidl_vec_removeAt(&model->operands, index);
for (Operation& operation : model->operations) {
removeValueAndDecrementGreaterValues(&operation.inputs, index);
removeValueAndDecrementGreaterValues(&operation.outputs, index);
}
removeValueAndDecrementGreaterValues(&model->inputIndexes, index);
removeValueAndDecrementGreaterValues(&model->outputIndexes, index);
}
static bool removeOperandSkip(size_t operand, const Model& model) {
for (const Operation& operation : model.operations) {
// Skip removeOperandTest for the following operations.
// - SPLIT's outputs are not checked during prepareModel.
if (operation.type == OperationType::SPLIT) {
for (const size_t outOprand : operation.outputs) {
if (operand == outOprand) {
return true;
}
}
}
// BIDIRECTIONAL_SEQUENCE_LSTM and BIDIRECTIONAL_SEQUENCE_RNN can have either one or two
// outputs depending on their mergeOutputs parameter.
if (operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_LSTM ||
operation.type == OperationType::BIDIRECTIONAL_SEQUENCE_RNN) {
for (const size_t outOprand : operation.outputs) {
if (operand == outOprand) {
return true;
}
}
}
}
return false;
}
static void removeOperandTest(const sp<IDevice>& device, const Model& model) {
for (size_t operand = 0; operand < model.operands.size(); ++operand) {
if (removeOperandSkip(operand, model)) {
continue;
}
const std::string message = "removeOperandTest: operand " + std::to_string(operand);
validate(device, message, model,
[operand](Model* model) { removeOperand(model, operand); });
}
}
///////////////////////// REMOVE OPERATION /////////////////////////
static void removeOperation(Model* model, uint32_t index) {
for (uint32_t operand : model->operations[index].inputs) {
model->operands[operand].numberOfConsumers--;
}
hidl_vec_removeAt(&model->operations, index);
}
static void removeOperationTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
const std::string message = "removeOperationTest: operation " + std::to_string(operation);
validate(device, message, model,
[operation](Model* model) { removeOperation(model, operation); });
}
}
///////////////////////// REMOVE OPERATION INPUT /////////////////////////
static bool removeOperationInputSkip(const Operation& op, size_t input) {
// Skip removeOperationInputTest for the following operations.
// - CONCATENATION has at least 2 inputs, with the last element being INT32.
// - CONV_2D, DEPTHWISE_CONV_2D, MAX_POOL_2D, AVERAGE_POOL_2D, L2_POOL_2D, RESIZE_BILINEAR,
// SPACE_TO_DEPTH, SPACE_TO_DEPTH, SPACE_TO_BATCH_ND, BATCH_TO_SPACE_ND can have an optional
// layout parameter.
// - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional axis
// parameter.
switch (op.type) {
case OperationType::CONCATENATION: {
if (op.inputs.size() > 2 && input != op.inputs.size() - 1) {
return true;
}
} break;
case OperationType::DEPTHWISE_CONV_2D: {
if ((op.inputs.size() == 12 && input == 11) || (op.inputs.size() == 9 && input == 8)) {
return true;
}
} break;
case OperationType::CONV_2D:
case OperationType::AVERAGE_POOL_2D:
case OperationType::MAX_POOL_2D:
case OperationType::L2_POOL_2D: {
if ((op.inputs.size() == 11 && input == 10) || (op.inputs.size() == 8 && input == 7)) {
return true;
}
} break;
case OperationType::RESIZE_BILINEAR: {
if (op.inputs.size() == 4 && input == 3) {
return true;
}
} break;
case OperationType::SPACE_TO_DEPTH:
case OperationType::DEPTH_TO_SPACE:
case OperationType::BATCH_TO_SPACE_ND: {
if (op.inputs.size() == 3 && input == 2) {
return true;
}
} break;
case OperationType::SPACE_TO_BATCH_ND: {
if (op.inputs.size() == 4 && input == 3) {
return true;
}
} break;
case OperationType::L2_NORMALIZATION: {
if (op.inputs.size() == 2 && input == 1) {
return true;
}
} break;
case OperationType::LOCAL_RESPONSE_NORMALIZATION: {
if (op.inputs.size() == 6 && input == 5) {
return true;
}
} break;
case OperationType::SOFTMAX: {
if (op.inputs.size() == 3 && input == 2) {
return true;
}
} break;
default:
break;
}
return false;
}
static void removeOperationInputTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
for (size_t input = 0; input < model.operations[operation].inputs.size(); ++input) {
const Operation& op = model.operations[operation];
if (removeOperationInputSkip(op, input)) {
continue;
}
const std::string message = "removeOperationInputTest: operation " +
std::to_string(operation) + ", input " +
std::to_string(input);
validate(device, message, model, [operation, input](Model* model) {
uint32_t operand = model->operations[operation].inputs[input];
model->operands[operand].numberOfConsumers--;
hidl_vec_removeAt(&model->operations[operation].inputs, input);
});
}
}
}
///////////////////////// REMOVE OPERATION OUTPUT /////////////////////////
static void removeOperationOutputTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
for (size_t output = 0; output < model.operations[operation].outputs.size(); ++output) {
const std::string message = "removeOperationOutputTest: operation " +
std::to_string(operation) + ", output " +
std::to_string(output);
validate(device, message, model, [operation, output](Model* model) {
hidl_vec_removeAt(&model->operations[operation].outputs, output);
});
}
}
}
///////////////////////// MODEL VALIDATION /////////////////////////
// TODO: remove model input
// TODO: remove model output
// TODO: add unused operation
///////////////////////// ADD OPERATION INPUT /////////////////////////
static bool addOperationInputSkip(const Operation& op) {
// Skip addOperationInputTest for the following operations.
// - L2_NORMALIZATION, LOCAL_RESPONSE_NORMALIZATION, SOFTMAX can have an optional INT32 axis
// parameter.
if ((op.type == OperationType::L2_NORMALIZATION && op.inputs.size() == 1) ||
(op.type == OperationType::LOCAL_RESPONSE_NORMALIZATION && op.inputs.size() == 5) ||
(op.type == OperationType::SOFTMAX && op.inputs.size() == 2)) {
return true;
}
return false;
}
static void addOperationInputTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
if (addOperationInputSkip(model.operations[operation])) {
continue;
}
const std::string message = "addOperationInputTest: operation " + std::to_string(operation);
validate(device, message, model, [operation](Model* model) {
uint32_t index = addOperand(model, OperandLifeTime::MODEL_INPUT);
hidl_vec_push_back(&model->operations[operation].inputs, index);
hidl_vec_push_back(&model->inputIndexes, index);
});
}
}
///////////////////////// ADD OPERATION OUTPUT /////////////////////////
static void addOperationOutputTest(const sp<IDevice>& device, const Model& model) {
for (size_t operation = 0; operation < model.operations.size(); ++operation) {
const std::string message =
"addOperationOutputTest: operation " + std::to_string(operation);
validate(device, message, model, [operation](Model* model) {
uint32_t index = addOperand(model, OperandLifeTime::MODEL_OUTPUT);
hidl_vec_push_back(&model->operations[operation].outputs, index);
hidl_vec_push_back(&model->outputIndexes, index);
});
}
}
///////////////////////// VALIDATE EXECUTION PREFERENCE /////////////////////////
static const int32_t invalidExecutionPreferences[] = {
static_cast<int32_t>(ExecutionPreference::LOW_POWER) - 1, // lower bound
static_cast<int32_t>(ExecutionPreference::SUSTAINED_SPEED) + 1, // upper bound
};
static void mutateExecutionPreferenceTest(const sp<IDevice>& device, const Model& model) {
for (int32_t preference : invalidExecutionPreferences) {
const std::string message =
"mutateExecutionPreferenceTest: preference " + std::to_string(preference);
validate(
device, message, model, [](Model*) {},
static_cast<ExecutionPreference>(preference));
}
}
////////////////////////// ENTRY POINT //////////////////////////////
void validateModel(const sp<IDevice>& device, const Model& model) {
mutateOperandTypeTest(device, model);
mutateOperandRankTest(device, model);
mutateOperandScaleTest(device, model);
mutateOperandZeroPointTest(device, model);
mutateOperationOperandTypeTest(device, model);
mutateOperationTypeTest(device, model);
mutateOperationInputOperandIndexTest(device, model);
mutateOperationOutputOperandIndexTest(device, model);
removeOperandTest(device, model);
removeOperationTest(device, model);
removeOperationInputTest(device, model);
removeOperationOutputTest(device, model);
addOperationInputTest(device, model);
addOperationOutputTest(device, model);
mutateExecutionPreferenceTest(device, model);
}
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

172
neuralnetworks/1.3/vts/functional/ValidateRequest.cpp
View file

@ -1,172 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "1.0/Utils.h"
#include "1.2/Callbacks.h"
#include "ExecutionBurstController.h"
#include "GeneratedTestHarness.h"
#include "TestHarness.h"
#include "Utils.h"
#include "VtsHalNeuralnetworks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using V1_0::ErrorStatus;
using V1_0::Request;
using V1_2::IPreparedModel;
using V1_2::MeasureTiming;
using V1_2::OutputShape;
using V1_2::Timing;
using V1_2::implementation::ExecutionCallback;
///////////////////////// UTILITY FUNCTIONS /////////////////////////
static bool badTiming(Timing timing) {
return timing.timeOnDevice == UINT64_MAX && timing.timeInDriver == UINT64_MAX;
}
// Primary validation function. This function will take a valid request, apply a
// mutation to it to invalidate the request, then pass it to interface calls
// that use the request. Note that the request here is passed by value, and any
// mutation to the request does not leave this function.
static void validate(const sp<IPreparedModel>& preparedModel, const std::string& message,
Request request, const std::function<void(Request*)>& mutation) {
mutation(&request);
// We'd like to test both with timing requested and without timing
// requested. Rather than running each test both ways, we'll decide whether
// to request timing by hashing the message. We do not use std::hash because
// it is not guaranteed stable across executions.
char hash = 0;
for (auto c : message) {
hash ^= c;
};
MeasureTiming measure = (hash & 1) ? MeasureTiming::YES : MeasureTiming::NO;
// asynchronous
{
SCOPED_TRACE(message + " [execute_1_2]");
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
Return<ErrorStatus> executeLaunchStatus =
preparedModel->execute_1_2(request, measure, executionCallback);
ASSERT_TRUE(executeLaunchStatus.isOk());
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, static_cast<ErrorStatus>(executeLaunchStatus));
executionCallback->wait();
ErrorStatus executionReturnStatus = executionCallback->getStatus();
const auto& outputShapes = executionCallback->getOutputShapes();
Timing timing = executionCallback->getTiming();
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, executionReturnStatus);
ASSERT_EQ(outputShapes.size(), 0);
ASSERT_TRUE(badTiming(timing));
}
// synchronous
{
SCOPED_TRACE(message + " [executeSynchronously]");
Return<void> executeStatus = preparedModel->executeSynchronously(
request, measure,
[](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes,
const Timing& timing) {
ASSERT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
EXPECT_EQ(outputShapes.size(), 0);
EXPECT_TRUE(badTiming(timing));
});
ASSERT_TRUE(executeStatus.isOk());
}
// burst
{
SCOPED_TRACE(message + " [burst]");
// create burst
std::shared_ptr<::android::nn::ExecutionBurstController> burst =
android::nn::ExecutionBurstController::create(preparedModel, /*blocking=*/true);
ASSERT_NE(nullptr, burst.get());
// create memory keys
std::vector<intptr_t> keys(request.pools.size());
for (size_t i = 0; i < keys.size(); ++i) {
keys[i] = reinterpret_cast<intptr_t>(&request.pools[i]);
}
// execute and verify
ErrorStatus error;
std::vector<OutputShape> outputShapes;
Timing timing;
std::tie(error, outputShapes, timing) = burst->compute(request, measure, keys);
EXPECT_EQ(ErrorStatus::INVALID_ARGUMENT, error);
EXPECT_EQ(outputShapes.size(), 0);
EXPECT_TRUE(badTiming(timing));
// additional burst testing
if (request.pools.size() > 0) {
// valid free
burst->freeMemory(keys.front());
// negative test: invalid free of unknown (blank) memory
burst->freeMemory(intptr_t{});
// negative test: double free of memory
burst->freeMemory(keys.front());
}
}
}
///////////////////////// REMOVE INPUT ////////////////////////////////////
static void removeInputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
for (size_t input = 0; input < request.inputs.size(); ++input) {
const std::string message = "removeInput: removed input " + std::to_string(input);
validate(preparedModel, message, request,
[input](Request* request) { hidl_vec_removeAt(&request->inputs, input); });
}
}
///////////////////////// REMOVE OUTPUT ////////////////////////////////////
static void removeOutputTest(const sp<IPreparedModel>& preparedModel, const Request& request) {
for (size_t output = 0; output < request.outputs.size(); ++output) {
const std::string message = "removeOutput: removed Output " + std::to_string(output);
validate(preparedModel, message, request,
[output](Request* request) { hidl_vec_removeAt(&request->outputs, output); });
}
}
///////////////////////////// ENTRY POINT //////////////////////////////////
void validateRequest(const sp<IPreparedModel>& preparedModel, const Request& request) {
removeInputTest(preparedModel, request);
removeOutputTest(preparedModel, request);
}
void validateRequestFailure(const sp<IPreparedModel>& preparedModel, const Request& request) {
SCOPED_TRACE("Expecting request to fail [executeSynchronously]");
Return<void> executeStatus = preparedModel->executeSynchronously(
request, MeasureTiming::NO,
[](ErrorStatus error, const hidl_vec<OutputShape>& outputShapes, const Timing& timing) {
ASSERT_NE(ErrorStatus::NONE, error);
EXPECT_EQ(outputShapes.size(), 0);
EXPECT_TRUE(badTiming(timing));
});
ASSERT_TRUE(executeStatus.isOk());
}
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

173
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.cpp
View file

@ -1,173 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define LOG_TAG "neuralnetworks_hidl_hal_test"
#include "VtsHalNeuralnetworks.h"
#include <android-base/logging.h>
#include <hidl/ServiceManagement.h>
#include <string>
#include <utility>
#include "1.0/Callbacks.h"
#include "1.0/Utils.h"
#include "GeneratedTestHarness.h"
#include "TestHarness.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using HidlToken =
hidl_array<uint8_t, static_cast<uint32_t>(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
using V1_0::ErrorStatus;
using V1_0::Request;
using V1_1::ExecutionPreference;
using V1_2::IPreparedModel;
using V1_2::implementation::PreparedModelCallback;
// internal helper function
void createPreparedModel(const sp<IDevice>& device, const Model& model,
sp<IPreparedModel>* preparedModel) {
ASSERT_NE(nullptr, preparedModel);
*preparedModel = nullptr;
// see if service can handle model
bool fullySupportsModel = false;
const Return<void> supportedCall = device->getSupportedOperations_1_3(
model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
ASSERT_EQ(ErrorStatus::NONE, status);
ASSERT_NE(0ul, supported.size());
fullySupportsModel = std::all_of(supported.begin(), supported.end(),
[](bool valid) { return valid; });
});
ASSERT_TRUE(supportedCall.isOk());
// launch prepare model
const sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
const Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_3(
model, ExecutionPreference::FAST_SINGLE_ANSWER, hidl_vec<hidl_handle>(),
hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
ASSERT_TRUE(prepareLaunchStatus.isOk());
ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
// retrieve prepared model
preparedModelCallback->wait();
const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
*preparedModel = getPreparedModel_1_2(preparedModelCallback);
// The getSupportedOperations_1_3 call returns a list of operations that are
// guaranteed not to fail if prepareModel_1_3 is called, and
// 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.
// If a driver has any doubt that it can prepare an operation, it must
// return false. So here, if a driver isn't sure if it can support an
// operation, but reports that it successfully prepared the model, the test
// can continue.
if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
ASSERT_EQ(nullptr, preparedModel->get());
LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare "
"model that it does not support.";
std::cout << "[ ] Early termination of test because vendor service cannot "
"prepare model that it does not support."
<< std::endl;
GTEST_SKIP();
}
ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);
ASSERT_NE(nullptr, preparedModel->get());
}
void NeuralnetworksHidlTest::SetUp() {
testing::TestWithParam<NeuralnetworksHidlTestParam>::SetUp();
ASSERT_NE(kDevice, nullptr);
}
static NamedDevice makeNamedDevice(const std::string& name) {
return {name, IDevice::getService(name)};
}
static std::vector<NamedDevice> getNamedDevicesImpl() {
// Retrieves the name of all service instances that implement IDevice,
// including any Lazy HAL instances.
const std::vector<std::string> names = hardware::getAllHalInstanceNames(IDevice::descriptor);
// Get a handle to each device and pair it with its name.
std::vector<NamedDevice> namedDevices;
namedDevices.reserve(names.size());
std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice);
return namedDevices;
}
const std::vector<NamedDevice>& getNamedDevices() {
const static std::vector<NamedDevice> devices = getNamedDevicesImpl();
return devices;
}
std::string printNeuralnetworksHidlTest(
const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info) {
return gtestCompliantName(getName(info.param));
}
INSTANTIATE_DEVICE_TEST(NeuralnetworksHidlTest);
// Forward declaration from ValidateModel.cpp
void validateModel(const sp<IDevice>& device, const Model& model);
// Forward declaration from ValidateRequest.cpp
void validateRequest(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
// Forward declaration from ValidateRequest.cpp
void validateRequestFailure(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
// Forward declaration from ValidateBurst.cpp
void validateBurst(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
void validateEverything(const sp<IDevice>& device, const Model& model, const Request& request) {
validateModel(device, model);
// Create IPreparedModel.
sp<IPreparedModel> preparedModel;
createPreparedModel(device, model, &preparedModel);
if (preparedModel == nullptr) return;
validateRequest(preparedModel, request);
validateBurst(preparedModel, request);
}
void validateFailure(const sp<IDevice>& device, const Model& model, const Request& request) {
// TODO: Should this always succeed?
// What if the invalid input is part of the model (i.e., a parameter).
validateModel(device, model);
// Create IPreparedModel.
sp<IPreparedModel> preparedModel;
createPreparedModel(device, model, &preparedModel);
if (preparedModel == nullptr) return;
validateRequestFailure(preparedModel, request);
}
TEST_P(ValidationTest, Test) {
const Model model = createModel(kTestModel);
const Request request = createRequest(kTestModel);
if (kTestModel.expectFailure) {
validateFailure(kDevice, model, request);
} else {
validateEverything(kDevice, model, request);
}
}
INSTANTIATE_GENERATED_TEST(ValidationTest, [](const test_helper::TestModel&) { return true; });
sp<IPreparedModel> getPreparedModel_1_2(const sp<PreparedModelCallback>& callback) {
sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel();
return IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);
}
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

58
neuralnetworks/1.3/vts/functional/VtsHalNeuralnetworks.h
View file

@ -1,58 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef ANDROID_HARDWARE_NEURALNETWORKS_V1_3_VTS_HAL_NEURALNETWORKS_H
#define ANDROID_HARDWARE_NEURALNETWORKS_V1_3_VTS_HAL_NEURALNETWORKS_H
#include <android/hardware/neuralnetworks/1.2/IPreparedModel.h>
#include <android/hardware/neuralnetworks/1.3/IDevice.h>
#include <android/hardware/neuralnetworks/1.3/types.h>
#include <gtest/gtest.h>
#include "1.0/Utils.h"
#include "1.2/Callbacks.h"
namespace android::hardware::neuralnetworks::V1_3::vts::functional {
using NamedDevice = Named<sp<IDevice>>;
using NeuralnetworksHidlTestParam = NamedDevice;
class NeuralnetworksHidlTest : public testing::TestWithParam<NeuralnetworksHidlTestParam> {
protected:
void SetUp() override;
const sp<IDevice> kDevice = getData(GetParam());
};
const std::vector<NamedDevice>& getNamedDevices();
std::string printNeuralnetworksHidlTest(
const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info);
#define INSTANTIATE_DEVICE_TEST(TestSuite) \
INSTANTIATE_TEST_SUITE_P(PerInstance, TestSuite, testing::ValuesIn(getNamedDevices()), \
printNeuralnetworksHidlTest)
// Create an IPreparedModel object. If the model cannot be prepared,
// "preparedModel" will be nullptr instead.
void createPreparedModel(const sp<IDevice>& device, const Model& model,
sp<V1_2::IPreparedModel>* preparedModel);
// Utility function to get PreparedModel from callback and downcast to V1_2.
sp<V1_2::IPreparedModel> getPreparedModel_1_2(
const sp<V1_2::implementation::PreparedModelCallback>& callback);
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional
#endif // ANDROID_HARDWARE_NEURALNETWORKS_V1_3_VTS_HAL_NEURALNETWORKS_H