tequilaOS/platform_hardware_interfaces
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Add NNAPI loop timeout VTS test" am: 2225653ff1

Browse source 
Change-Id: I952c5db6a279173ae91d8791f0cd8111210dd662
This commit is contained in:
Automerger Merge Worker 2020-03-05 17:27:21 +00:00
commit fbcf4ca055
2 changed files with 55 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.cpp
View file

@ -74,7 +74,7 @@ namespace {
enum class Executor { ASYNC, SYNC, BURST, FENCED };
enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT };
enum class OutputType { FULLY_SPECIFIED, UNSPECIFIED, INSUFFICIENT, MISSED_DEADLINE };
enum class MemoryType { SHARED, DEVICE };
@ -495,16 +495,18 @@ static std::vector<TestBuffer> getOutputBuffers(const TestModel& testModel, cons
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
const OptionalTimeoutDuration& loopTimeoutDuration,
sp<ExecutionCallback>& callback) {
return preparedModel->execute_1_3(request, measure, {}, {}, callback);
return preparedModel->execute_1_3(request, measure, {}, loopTimeoutDuration, callback);
}
static Return<ErrorStatus> ExecutePreparedModel(const sp<IPreparedModel>& preparedModel,
const Request& request, MeasureTiming measure,
const OptionalTimeoutDuration& loopTimeoutDuration,
hidl_vec<OutputShape>* outputShapes,
Timing* timing) {
ErrorStatus result;
Return<void> ret = preparedModel->executeSynchronously_1_3(
request, measure, {}, {},
request, measure, {}, loopTimeoutDuration,
[&result, outputShapes, timing](ErrorStatus error, const hidl_vec<OutputShape>& shapes,
const Timing& time) {
result = error;
@ -545,6 +547,17 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
makeOutputInsufficientSize(/*outputIndex=*/0, &request);
}
OptionalTimeoutDuration loopTimeoutDuration;
// OutputType::MISSED_DEADLINE is only used by
// TestKind::INTINITE_LOOP_TIMEOUT tests to verify that an infinite loop is
// aborted after a timeout.
if (testConfig.outputType == OutputType::MISSED_DEADLINE) {
// Override the default loop timeout duration with a small value to
// speed up test execution.
constexpr uint64_t kMillisecond = 1'000'000;
loopTimeoutDuration.nanoseconds(1 * kMillisecond);
}
ErrorStatus executionStatus;
hidl_vec<OutputShape> outputShapes;
Timing timing;
@ -554,8 +567,9 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
// launch execution
sp<ExecutionCallback> executionCallback = new ExecutionCallback();
Return<ErrorStatus> executionLaunchStatus = ExecutePreparedModel(
preparedModel, request, testConfig.measureTiming, executionCallback);
Return<ErrorStatus> executionLaunchStatus =
ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
loopTimeoutDuration, executionCallback);
ASSERT_TRUE(executionLaunchStatus.isOk());
EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus));
@ -571,8 +585,9 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
SCOPED_TRACE("synchronous");
// execute
Return<ErrorStatus> executionReturnStatus = ExecutePreparedModel(
preparedModel, request, testConfig.measureTiming, &outputShapes, &timing);
Return<ErrorStatus> executionReturnStatus =
ExecutePreparedModel(preparedModel, request, testConfig.measureTiming,
loopTimeoutDuration, &outputShapes, &timing);
ASSERT_TRUE(executionReturnStatus.isOk());
executionStatus = static_cast<ErrorStatus>(executionReturnStatus);
@ -612,7 +627,7 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
hidl_handle syncFenceHandle;
sp<IFencedExecutionCallback> fencedCallback;
Return<void> ret = preparedModel->executeFenced(
request, {}, testConfig.measureTiming, {}, {}, {},
request, {}, testConfig.measureTiming, {}, loopTimeoutDuration, {},
[&result, &syncFenceHandle, &fencedCallback](
ErrorStatus error, const hidl_handle& handle,
const sp<IFencedExecutionCallback>& callback) {
@ -686,6 +701,11 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
ASSERT_EQ(outputShapes.size(), testModel.main.outputIndexes.size());
ASSERT_FALSE(outputShapes[0].isSufficient);
return;
case OutputType::MISSED_DEADLINE:
ASSERT_TRUE(executionStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT ||
executionStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT)
<< "executionStatus = " << executionStatus;
return;
}
// Go through all outputs, check returned output shapes.
@ -736,6 +756,12 @@ void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>&
LOG(FATAL) << "Wrong TestKind for EvaluatePreparedModel";
return;
} break;
case TestKind::INTINITE_LOOP_TIMEOUT: {
outputTypesList = {OutputType::MISSED_DEADLINE};
measureTimingList = {MeasureTiming::NO, MeasureTiming::YES};
// Burst does not support V1_3 loop timeout.
executorList = {Executor::ASYNC, Executor::SYNC, Executor::FENCED};
} break;
}
for (const OutputType outputType : outputTypesList) {
@ -794,7 +820,8 @@ void Execute(const sp<IDevice>& device, const TestModel& testModel, TestKind tes
case TestKind::GENERAL:
case TestKind::DYNAMIC_SHAPE:
case TestKind::MEMORY_DOMAIN:
case TestKind::FENCED_COMPUTE: {
case TestKind::FENCED_COMPUTE:
case TestKind::INTINITE_LOOP_TIMEOUT: {
createPreparedModel(device, model, &preparedModel);
if (preparedModel == nullptr) return;
EvaluatePreparedModel(device, preparedModel, testModel, testKind);
@ -863,24 +890,31 @@ class FencedComputeTest : public GeneratedTest {};
// Tag for the dynamic output shape tests
class QuantizationCouplingTest : public GeneratedTest {};
// Tag for the loop timeout tests
class InfiniteLoopTimeoutTest : public GeneratedTest {};
TEST_P(GeneratedTest, Test) {
Execute(kDevice, kTestModel, /*testKind=*/TestKind::GENERAL);
Execute(kDevice, kTestModel, TestKind::GENERAL);
}
TEST_P(DynamicOutputShapeTest, Test) {
Execute(kDevice, kTestModel, /*testKind=*/TestKind::DYNAMIC_SHAPE);
Execute(kDevice, kTestModel, TestKind::DYNAMIC_SHAPE);
}
TEST_P(MemoryDomainTest, Test) {
Execute(kDevice, kTestModel, /*testKind=*/TestKind::MEMORY_DOMAIN);
Execute(kDevice, kTestModel, TestKind::MEMORY_DOMAIN);
}
TEST_P(FencedComputeTest, Test) {
Execute(kDevice, kTestModel, /*testKind=*/TestKind::FENCED_COMPUTE);
Execute(kDevice, kTestModel, TestKind::FENCED_COMPUTE);
}
TEST_P(QuantizationCouplingTest, Test) {
Execute(kDevice, kTestModel, /*testKind=*/TestKind::QUANTIZATION_COUPLING);
Execute(kDevice, kTestModel, TestKind::QUANTIZATION_COUPLING);
}
TEST_P(InfiniteLoopTimeoutTest, Test) {
Execute(kDevice, kTestModel, TestKind::INTINITE_LOOP_TIMEOUT);
}
INSTANTIATE_GENERATED_TEST(GeneratedTest,
@ -900,4 +934,8 @@ INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testMod
return testModel.hasQuant8CoupledOperands() && testModel.main.operations.size() == 1;
});
INSTANTIATE_GENERATED_TEST(InfiniteLoopTimeoutTest, [](const TestModel& testModel) {
return testModel.isInfiniteLoopTimeoutTest();
});
} // namespace android::hardware::neuralnetworks::V1_3::vts::functional

4
neuralnetworks/1.3/vts/functional/GeneratedTestHarness.h
View file

@ -70,7 +70,9 @@ enum class TestKind {
// Tests if quantized model with TENSOR_QUANT8_ASYMM produces the same result
// (OK/SKIPPED/FAILED) as the model with all such tensors converted to
// TENSOR_QUANT8_ASYMM_SIGNED.
QUANTIZATION_COUPLING
QUANTIZATION_COUPLING,
// Runs a test model and verifies that MISSED_DEADLINE_* is returned.
INTINITE_LOOP_TIMEOUT
};
void EvaluatePreparedModel(const sp<IDevice>& device, const sp<IPreparedModel>& preparedModel,