mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
506 lines
22 KiB
506 lines
22 KiB
// |
|
// Copyright © 2017 Arm Ltd. All rights reserved. |
|
// SPDX-License-Identifier: MIT |
|
// |
|
|
|
#include "../ImageTensorGenerator/ImageTensorGenerator.hpp" |
|
#include "../InferenceTest.hpp" |
|
#include "ModelAccuracyChecker.hpp" |
|
#include "armnnDeserializer/IDeserializer.hpp" |
|
|
|
#include <armnnUtils/Filesystem.hpp> |
|
#include <armnnUtils/TContainer.hpp> |
|
|
|
#include <cxxopts/cxxopts.hpp> |
|
#include <map> |
|
|
|
using namespace armnn::test; |
|
|
|
/** Load image names and ground-truth labels from the image directory and the ground truth label file |
|
* |
|
* @pre \p validationLabelPath exists and is valid regular file |
|
* @pre \p imageDirectoryPath exists and is valid directory |
|
* @pre labels in validation file correspond to images which are in lexicographical order with the image name |
|
* @pre image index starts at 1 |
|
* @pre \p begIndex and \p endIndex are end-inclusive |
|
* |
|
* @param[in] validationLabelPath Path to validation label file |
|
* @param[in] imageDirectoryPath Path to directory containing validation images |
|
* @param[in] begIndex Begin index of images to be loaded. Inclusive |
|
* @param[in] endIndex End index of images to be loaded. Inclusive |
|
* @param[in] excludelistPath Path to excludelist file |
|
* @return A map mapping image file names to their corresponding ground-truth labels |
|
*/ |
|
map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath, |
|
const string& imageDirectoryPath, |
|
size_t begIndex = 0, |
|
size_t endIndex = 0, |
|
const string& excludelistPath = ""); |
|
|
|
/** Load model output labels from file |
|
* |
|
* @pre \p modelOutputLabelsPath exists and is a regular file |
|
* |
|
* @param[in] modelOutputLabelsPath path to model output labels file |
|
* @return A vector of labels, which in turn is described by a list of category names |
|
*/ |
|
std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath); |
|
|
|
int main(int argc, char* argv[]) |
|
{ |
|
try |
|
{ |
|
armnn::LogSeverity level = armnn::LogSeverity::Debug; |
|
armnn::ConfigureLogging(true, true, level); |
|
|
|
std::string modelPath; |
|
std::string modelFormat; |
|
std::vector<std::string> inputNames; |
|
std::vector<std::string> outputNames; |
|
std::string dataDir; |
|
std::string modelOutputLabelsPath; |
|
std::string validationLabelPath; |
|
std::string inputLayout; |
|
std::vector<armnn::BackendId> computeDevice; |
|
std::string validationRange; |
|
std::string excludelistPath; |
|
|
|
const std::string backendsMessage = "Which device to run layers on by default. Possible choices: " |
|
+ armnn::BackendRegistryInstance().GetBackendIdsAsString(); |
|
|
|
try |
|
{ |
|
cxxopts::Options options("ModeAccuracyTool-Armnn","Options"); |
|
|
|
options.add_options() |
|
("h,help", "Display help messages") |
|
("m,model-path", |
|
"Path to armnn format model file", |
|
cxxopts::value<std::string>(modelPath)) |
|
("f,model-format", |
|
"The model format. Supported values: tflite", |
|
cxxopts::value<std::string>(modelFormat)) |
|
("i,input-name", |
|
"Identifier of the input tensors in the network separated by comma with no space.", |
|
cxxopts::value<std::vector<std::string>>(inputNames)) |
|
("o,output-name", |
|
"Identifier of the output tensors in the network separated by comma with no space.", |
|
cxxopts::value<std::vector<std::string>>(outputNames)) |
|
("d,data-dir", |
|
"Path to directory containing the ImageNet test data", |
|
cxxopts::value<std::string>(dataDir)) |
|
("p,model-output-labels", |
|
"Path to model output labels file.", |
|
cxxopts::value<std::string>(modelOutputLabelsPath)) |
|
("v,validation-labels-path", |
|
"Path to ImageNet Validation Label file", |
|
cxxopts::value<std::string>(validationLabelPath)) |
|
("l,data-layout", |
|
"Data layout. Supported value: NHWC, NCHW. Default: NHWC", |
|
cxxopts::value<std::string>(inputLayout)->default_value("NHWC")) |
|
("c,compute", |
|
backendsMessage.c_str(), |
|
cxxopts::value<std::vector<armnn::BackendId>>(computeDevice)->default_value("CpuAcc,CpuRef")) |
|
("r,validation-range", |
|
"The range of the images to be evaluated. Specified in the form <begin index>:<end index>." |
|
"The index starts at 1 and the range is inclusive." |
|
"By default the evaluation will be performed on all images.", |
|
cxxopts::value<std::string>(validationRange)->default_value("1:0")) |
|
("e,excludelist-path", |
|
"Path to a excludelist file where each line denotes the index of an image to be " |
|
"excluded from evaluation.", |
|
cxxopts::value<std::string>(excludelistPath)->default_value("")); |
|
ARMNN_DEPRECATED_MSG_REMOVAL_DATE("This b,blacklist-path command is deprecated", "22.08") |
|
("b,blacklist-path", |
|
"Path to a blacklist file where each line denotes the index of an image to be " |
|
"excluded from evaluation. This command will be deprecated in favor of: --excludelist-path ", |
|
cxxopts::value<std::string>(excludelistPath)->default_value("")); |
|
|
|
auto result = options.parse(argc, argv); |
|
|
|
if (result.count("help") > 0) |
|
{ |
|
std::cout << options.help() << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
|
|
// Check for mandatory single options. |
|
std::string mandatorySingleParameters[] = { "model-path", "model-format", "input-name", "output-name", |
|
"data-dir", "model-output-labels", "validation-labels-path" }; |
|
for (auto param : mandatorySingleParameters) |
|
{ |
|
if (result.count(param) != 1) |
|
{ |
|
std::cerr << "Parameter \'--" << param << "\' is required but missing." << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
} |
|
} |
|
catch (const cxxopts::OptionException& e) |
|
{ |
|
std::cerr << e.what() << std::endl << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
catch (const std::exception& e) |
|
{ |
|
ARMNN_ASSERT_MSG(false, "Caught unexpected exception"); |
|
std::cerr << "Fatal internal error: " << e.what() << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
|
|
// Check if the requested backend are all valid |
|
std::string invalidBackends; |
|
if (!CheckRequestedBackendsAreValid(computeDevice, armnn::Optional<std::string&>(invalidBackends))) |
|
{ |
|
ARMNN_LOG(fatal) << "The list of preferred devices contains invalid backend IDs: " |
|
<< invalidBackends; |
|
return EXIT_FAILURE; |
|
} |
|
armnn::Status status; |
|
|
|
// Create runtime |
|
armnn::IRuntime::CreationOptions options; |
|
armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options)); |
|
std::ifstream file(modelPath); |
|
|
|
// Create Parser |
|
using IParser = armnnDeserializer::IDeserializer; |
|
auto armnnparser(IParser::Create()); |
|
|
|
// Create a network |
|
armnn::INetworkPtr network = armnnparser->CreateNetworkFromBinary(file); |
|
|
|
// Optimizes the network. |
|
armnn::IOptimizedNetworkPtr optimizedNet(nullptr, nullptr); |
|
try |
|
{ |
|
optimizedNet = armnn::Optimize(*network, computeDevice, runtime->GetDeviceSpec()); |
|
} |
|
catch (const armnn::Exception& e) |
|
{ |
|
std::stringstream message; |
|
message << "armnn::Exception (" << e.what() << ") caught from optimize."; |
|
ARMNN_LOG(fatal) << message.str(); |
|
return EXIT_FAILURE; |
|
} |
|
|
|
// Loads the network into the runtime. |
|
armnn::NetworkId networkId; |
|
status = runtime->LoadNetwork(networkId, std::move(optimizedNet)); |
|
if (status == armnn::Status::Failure) |
|
{ |
|
ARMNN_LOG(fatal) << "armnn::IRuntime: Failed to load network"; |
|
return EXIT_FAILURE; |
|
} |
|
|
|
// Set up Network |
|
using BindingPointInfo = InferenceModelInternal::BindingPointInfo; |
|
|
|
// Handle inputNames and outputNames, there can be multiple. |
|
std::vector<BindingPointInfo> inputBindings; |
|
for(auto& input: inputNames) |
|
{ |
|
const armnnDeserializer::BindingPointInfo& |
|
inputBindingInfo = armnnparser->GetNetworkInputBindingInfo(0, input); |
|
|
|
std::pair<armnn::LayerBindingId, armnn::TensorInfo> |
|
m_InputBindingInfo(inputBindingInfo.m_BindingId, inputBindingInfo.m_TensorInfo); |
|
inputBindings.push_back(m_InputBindingInfo); |
|
} |
|
|
|
std::vector<BindingPointInfo> outputBindings; |
|
for(auto& output: outputNames) |
|
{ |
|
const armnnDeserializer::BindingPointInfo& |
|
outputBindingInfo = armnnparser->GetNetworkOutputBindingInfo(0, output); |
|
|
|
std::pair<armnn::LayerBindingId, armnn::TensorInfo> |
|
m_OutputBindingInfo(outputBindingInfo.m_BindingId, outputBindingInfo.m_TensorInfo); |
|
outputBindings.push_back(m_OutputBindingInfo); |
|
} |
|
|
|
// Load model output labels |
|
if (modelOutputLabelsPath.empty() || !fs::exists(modelOutputLabelsPath) || |
|
!fs::is_regular_file(modelOutputLabelsPath)) |
|
{ |
|
ARMNN_LOG(fatal) << "Invalid model output labels path at " << modelOutputLabelsPath; |
|
} |
|
const std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels = |
|
LoadModelOutputLabels(modelOutputLabelsPath); |
|
|
|
// Parse begin and end image indices |
|
std::vector<std::string> imageIndexStrs = armnnUtils::SplitBy(validationRange, ":"); |
|
size_t imageBegIndex; |
|
size_t imageEndIndex; |
|
if (imageIndexStrs.size() != 2) |
|
{ |
|
ARMNN_LOG(fatal) << "Invalid validation range specification: Invalid format " << validationRange; |
|
return EXIT_FAILURE; |
|
} |
|
try |
|
{ |
|
imageBegIndex = std::stoul(imageIndexStrs[0]); |
|
imageEndIndex = std::stoul(imageIndexStrs[1]); |
|
} |
|
catch (const std::exception& e) |
|
{ |
|
ARMNN_LOG(fatal) << "Invalid validation range specification: " << validationRange; |
|
return EXIT_FAILURE; |
|
} |
|
|
|
// Validate excludelist file if it's specified |
|
if (!excludelistPath.empty() && |
|
!(fs::exists(excludelistPath) && fs::is_regular_file(excludelistPath))) |
|
{ |
|
ARMNN_LOG(fatal) << "Invalid path to excludelist file at " << excludelistPath; |
|
return EXIT_FAILURE; |
|
} |
|
|
|
fs::path pathToDataDir(dataDir); |
|
const map<std::string, std::string> imageNameToLabel = LoadValidationImageFilenamesAndLabels( |
|
validationLabelPath, pathToDataDir.string(), imageBegIndex, imageEndIndex, excludelistPath); |
|
armnnUtils::ModelAccuracyChecker checker(imageNameToLabel, modelOutputLabels); |
|
|
|
if (ValidateDirectory(dataDir)) |
|
{ |
|
InferenceModel<armnnDeserializer::IDeserializer, float>::Params params; |
|
|
|
params.m_ModelPath = modelPath; |
|
params.m_IsModelBinary = true; |
|
params.m_ComputeDevices = computeDevice; |
|
// Insert inputNames and outputNames into params vector |
|
params.m_InputBindings.insert(std::end(params.m_InputBindings), |
|
std::begin(inputNames), |
|
std::end(inputNames)); |
|
params.m_OutputBindings.insert(std::end(params.m_OutputBindings), |
|
std::begin(outputNames), |
|
std::end(outputNames)); |
|
|
|
using TParser = armnnDeserializer::IDeserializer; |
|
// If dynamicBackends is empty it will be disabled by default. |
|
InferenceModel<TParser, float> model(params, false, ""); |
|
|
|
// Get input tensor information |
|
const armnn::TensorInfo& inputTensorInfo = model.GetInputBindingInfo().second; |
|
const armnn::TensorShape& inputTensorShape = inputTensorInfo.GetShape(); |
|
const armnn::DataType& inputTensorDataType = inputTensorInfo.GetDataType(); |
|
armnn::DataLayout inputTensorDataLayout; |
|
if (inputLayout == "NCHW") |
|
{ |
|
inputTensorDataLayout = armnn::DataLayout::NCHW; |
|
} |
|
else if (inputLayout == "NHWC") |
|
{ |
|
inputTensorDataLayout = armnn::DataLayout::NHWC; |
|
} |
|
else |
|
{ |
|
ARMNN_LOG(fatal) << "Invalid Data layout: " << inputLayout; |
|
return EXIT_FAILURE; |
|
} |
|
const unsigned int inputTensorWidth = |
|
inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[3] : inputTensorShape[2]; |
|
const unsigned int inputTensorHeight = |
|
inputTensorDataLayout == armnn::DataLayout::NCHW ? inputTensorShape[2] : inputTensorShape[1]; |
|
// Get output tensor info |
|
const unsigned int outputNumElements = model.GetOutputSize(); |
|
// Check output tensor shape is valid |
|
if (modelOutputLabels.size() != outputNumElements) |
|
{ |
|
ARMNN_LOG(fatal) << "Number of output elements: " << outputNumElements |
|
<< " , mismatches the number of output labels: " << modelOutputLabels.size(); |
|
return EXIT_FAILURE; |
|
} |
|
|
|
const unsigned int batchSize = 1; |
|
// Get normalisation parameters |
|
SupportedFrontend modelFrontend; |
|
if (modelFormat == "tflite") |
|
{ |
|
modelFrontend = SupportedFrontend::TFLite; |
|
} |
|
else |
|
{ |
|
ARMNN_LOG(fatal) << "Unsupported frontend: " << modelFormat; |
|
return EXIT_FAILURE; |
|
} |
|
const NormalizationParameters& normParams = GetNormalizationParameters(modelFrontend, inputTensorDataType); |
|
for (const auto& imageEntry : imageNameToLabel) |
|
{ |
|
const std::string imageName = imageEntry.first; |
|
std::cout << "Processing image: " << imageName << "\n"; |
|
|
|
vector<armnnUtils::TContainer> inputDataContainers; |
|
vector<armnnUtils::TContainer> outputDataContainers; |
|
|
|
auto imagePath = pathToDataDir / fs::path(imageName); |
|
switch (inputTensorDataType) |
|
{ |
|
case armnn::DataType::Signed32: |
|
inputDataContainers.push_back( |
|
PrepareImageTensor<int>(imagePath.string(), |
|
inputTensorWidth, inputTensorHeight, |
|
normParams, |
|
batchSize, |
|
inputTensorDataLayout)); |
|
outputDataContainers = { vector<int>(outputNumElements) }; |
|
break; |
|
case armnn::DataType::QAsymmU8: |
|
inputDataContainers.push_back( |
|
PrepareImageTensor<uint8_t>(imagePath.string(), |
|
inputTensorWidth, inputTensorHeight, |
|
normParams, |
|
batchSize, |
|
inputTensorDataLayout)); |
|
outputDataContainers = { vector<uint8_t>(outputNumElements) }; |
|
break; |
|
case armnn::DataType::Float32: |
|
default: |
|
inputDataContainers.push_back( |
|
PrepareImageTensor<float>(imagePath.string(), |
|
inputTensorWidth, inputTensorHeight, |
|
normParams, |
|
batchSize, |
|
inputTensorDataLayout)); |
|
outputDataContainers = { vector<float>(outputNumElements) }; |
|
break; |
|
} |
|
|
|
status = runtime->EnqueueWorkload(networkId, |
|
armnnUtils::MakeInputTensors(inputBindings, inputDataContainers), |
|
armnnUtils::MakeOutputTensors(outputBindings, outputDataContainers)); |
|
|
|
if (status == armnn::Status::Failure) |
|
{ |
|
ARMNN_LOG(fatal) << "armnn::IRuntime: Failed to enqueue workload for image: " << imageName; |
|
} |
|
|
|
checker.AddImageResult<armnnUtils::TContainer>(imageName, outputDataContainers); |
|
} |
|
} |
|
else |
|
{ |
|
return EXIT_SUCCESS; |
|
} |
|
|
|
for(unsigned int i = 1; i <= 5; ++i) |
|
{ |
|
std::cout << "Top " << i << " Accuracy: " << checker.GetAccuracy(i) << "%" << "\n"; |
|
} |
|
|
|
ARMNN_LOG(info) << "Accuracy Tool ran successfully!"; |
|
return EXIT_SUCCESS; |
|
} |
|
catch (const armnn::Exception& e) |
|
{ |
|
// Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an |
|
// exception of type std::length_error. |
|
// Using stderr instead in this context as there is no point in nesting try-catch blocks here. |
|
std::cerr << "Armnn Error: " << e.what() << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
catch (const std::exception& e) |
|
{ |
|
// Coverity fix: various boost exceptions can be thrown by methods called by this test. |
|
std::cerr << "WARNING: ModelAccuracyTool-Armnn: An error has occurred when running the " |
|
"Accuracy Tool: " << e.what() << std::endl; |
|
return EXIT_FAILURE; |
|
} |
|
} |
|
|
|
map<std::string, std::string> LoadValidationImageFilenamesAndLabels(const string& validationLabelPath, |
|
const string& imageDirectoryPath, |
|
size_t begIndex, |
|
size_t endIndex, |
|
const string& excludelistPath) |
|
{ |
|
// Populate imageFilenames with names of all .JPEG, .PNG images |
|
std::vector<std::string> imageFilenames; |
|
for (const auto& imageEntry : fs::directory_iterator(fs::path(imageDirectoryPath))) |
|
{ |
|
fs::path imagePath = imageEntry.path(); |
|
|
|
// Get extension and convert to uppercase |
|
std::string imageExtension = imagePath.extension().string(); |
|
std::transform(imageExtension.begin(), imageExtension.end(), imageExtension.begin(), ::toupper); |
|
|
|
if (fs::is_regular_file(imagePath) && (imageExtension == ".JPEG" || imageExtension == ".PNG")) |
|
{ |
|
imageFilenames.push_back(imagePath.filename().string()); |
|
} |
|
} |
|
if (imageFilenames.empty()) |
|
{ |
|
throw armnn::Exception("No image file (JPEG, PNG) found at " + imageDirectoryPath); |
|
} |
|
|
|
// Sort the image filenames lexicographically |
|
std::sort(imageFilenames.begin(), imageFilenames.end()); |
|
|
|
std::cout << imageFilenames.size() << " images found at " << imageDirectoryPath << std::endl; |
|
|
|
// Get default end index |
|
if (begIndex < 1 || endIndex > imageFilenames.size()) |
|
{ |
|
throw armnn::Exception("Invalid image index range"); |
|
} |
|
endIndex = endIndex == 0 ? imageFilenames.size() : endIndex; |
|
if (begIndex > endIndex) |
|
{ |
|
throw armnn::Exception("Invalid image index range"); |
|
} |
|
|
|
// Load excludelist if there is one |
|
std::vector<unsigned int> excludelist; |
|
if (!excludelistPath.empty()) |
|
{ |
|
std::ifstream excludelistFile(excludelistPath); |
|
unsigned int index; |
|
while (excludelistFile >> index) |
|
{ |
|
excludelist.push_back(index); |
|
} |
|
} |
|
|
|
// Load ground truth labels and pair them with corresponding image names |
|
std::string classification; |
|
map<std::string, std::string> imageNameToLabel; |
|
ifstream infile(validationLabelPath); |
|
size_t imageIndex = begIndex; |
|
size_t excludelistIndexCount = 0; |
|
while (std::getline(infile, classification)) |
|
{ |
|
if (imageIndex > endIndex) |
|
{ |
|
break; |
|
} |
|
// If current imageIndex is included in excludelist, skip the current image |
|
if (excludelistIndexCount < excludelist.size() && imageIndex == excludelist[excludelistIndexCount]) |
|
{ |
|
++imageIndex; |
|
++excludelistIndexCount; |
|
continue; |
|
} |
|
imageNameToLabel.insert(std::pair<std::string, std::string>(imageFilenames[imageIndex - 1], classification)); |
|
++imageIndex; |
|
} |
|
std::cout << excludelistIndexCount << " images in excludelist" << std::endl; |
|
std::cout << imageIndex - begIndex - excludelistIndexCount << " images to be loaded" << std::endl; |
|
return imageNameToLabel; |
|
} |
|
|
|
std::vector<armnnUtils::LabelCategoryNames> LoadModelOutputLabels(const std::string& modelOutputLabelsPath) |
|
{ |
|
std::vector<armnnUtils::LabelCategoryNames> modelOutputLabels; |
|
ifstream modelOutputLablesFile(modelOutputLabelsPath); |
|
std::string line; |
|
while (std::getline(modelOutputLablesFile, line)) |
|
{ |
|
armnnUtils::LabelCategoryNames tokens = armnnUtils::SplitBy(line, ":"); |
|
armnnUtils::LabelCategoryNames predictionCategoryNames = armnnUtils::SplitBy(tokens.back(), ","); |
|
std::transform(predictionCategoryNames.begin(), predictionCategoryNames.end(), predictionCategoryNames.begin(), |
|
[](const std::string& category) { return armnnUtils::Strip(category); }); |
|
modelOutputLabels.push_back(predictionCategoryNames); |
|
} |
|
return modelOutputLabels; |
|
} |