mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-16 01:33:39 +00:00
ORT fuzz testing (#5771)
* Added fuzz testing using ORT model. * The onnxruntime_security_fuzz driver code should accept either ONNX or ORT (based on the file extension) input file if /f flag is provided. * Added ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild test. * Added win-ci-fuzz-testing.yml to run build pipeline. * Prevent out-of-range access in the graph.cpp.
This commit is contained in:
satyajandhyala
GitHub
parent
84c1340f9b
commit
b495ae8103
6 changed files with 622 additions and 489 deletions
|
|
@ -3732,6 +3732,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr
|
|||
ORT_RETURN_IF(nullptr == fbs_node, "Node is missing. Invalid ORT format model.");
|
||||
std::unique_ptr<Node> node;
|
||||
ORT_RETURN_IF_ERROR(Node::LoadFromOrtFormat(*fbs_node, *this, logger_, node));
|
||||
ORT_RETURN_IF(node->Index() >= fbs_graph.max_node_index(), "Node index is out of range");
|
||||
nodes_[node->Index()] = std::move(node);
|
||||
++num_of_nodes_;
|
||||
}
|
||||
|
|
@ -3742,6 +3743,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr
|
|||
if (fbs_node_edges != nullptr) {
|
||||
for (const auto* fbs_node_edge : *fbs_node_edges) {
|
||||
ORT_RETURN_IF(nullptr == fbs_node_edge, "NodeEdge is missing. Invalid ORT format model.");
|
||||
ORT_RETURN_IF(fbs_node_edge->node_index() >= fbs_graph.max_node_index(), "Node index is out of range");
|
||||
ORT_RETURN_IF_ERROR(nodes_[fbs_node_edge->node_index()]->LoadEdgesFromOrtFormat(*fbs_node_edge, *this));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -237,6 +237,41 @@ static void DumpOrtModelAsJson(const std::string& model_uri) {
|
|||
std::ofstream(model_uri + ".json") << json;
|
||||
}
|
||||
*/
|
||||
/* The full build was causing the following error because the graph node array has some empyt (blank) node at some indices for certain ORT designs
|
||||
onnx runtime exception : Satisfied, but should not be : node == nullptr
|
||||
session_state.cc : 814 onnxruntime::SessionState::LoadFromOrtFormatCan't find node with index 4. Invalid ORT format model.
|
||||
The bug was due to loading an ORT format model in a full build, allowing optimizers to run, but trying to use the saved kernel information.
|
||||
As the optimizer removed some leaving gaps in the graph node vector.
|
||||
The build has been fixed in InferenceSession code. The following test case to catch this error.
|
||||
*/
|
||||
|
||||
TEST(OrtModelOnlyTests, ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild) {
|
||||
const std::basic_string<ORTCHAR_T> ort_file = ORT_TSTR("mnist.onnx.ort");
|
||||
SaveAndCompareModels("testdata/mnist.onnx", ort_file);
|
||||
|
||||
// DumpOrtModelAsJson(ToMBString(ort_file));
|
||||
|
||||
OrtModelTestInfo test_info;
|
||||
test_info.model_filename = ort_file;
|
||||
test_info.logid = "ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild";
|
||||
test_info.configs.push_back(std::make_pair(kOrtSessionOptionsConfigLoadModelFormat, "ORT"));
|
||||
|
||||
OrtValue ml_value;
|
||||
vector<float> data(28*28, 0.0);
|
||||
CreateMLValue<float>(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), {1,1,28,28}, data,
|
||||
&ml_value);
|
||||
test_info.inputs.insert(std::make_pair("Input3", ml_value));
|
||||
|
||||
// prepare outputs
|
||||
test_info.output_names = {"Plus214_Output_0"};
|
||||
test_info.output_verifier = [](const std::vector<OrtValue>& fetches) {
|
||||
const auto& output = fetches[0].Get<Tensor>();
|
||||
ASSERT_TRUE(output.Shape().NumDimensions() == 2);
|
||||
// ASSERT_TRUE(output.Data<float>()[0] == 125.f);
|
||||
};
|
||||
|
||||
RunOrtModel(test_info);
|
||||
}
|
||||
|
||||
TEST(OrtModelOnlyTests, SerializeToOrtFormat) {
|
||||
const std::basic_string<ORTCHAR_T> ort_file = ORT_TSTR("ort_github_issue_4031.onnx.ort");
|
||||
|
|
|
|||
|
|
@ -25,82 +25,74 @@
|
|||
|
||||
#include "testlog.h"
|
||||
|
||||
class OnnxPrediction
|
||||
{
|
||||
class OnnxPrediction {
|
||||
friend std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred);
|
||||
friend Logger::TestLog& Logger::TestLog::operator<<(OnnxPrediction& pred);
|
||||
public:
|
||||
using InputGeneratorFunctionType = std::function<void(OnnxPrediction&,
|
||||
size_t, const std::string&,
|
||||
ONNXTensorElementDataType,
|
||||
size_t, size_t)>;
|
||||
|
||||
public:
|
||||
public:
|
||||
using InputGeneratorFunctionType = std::function<void(OnnxPrediction&,
|
||||
size_t, const std::string&,
|
||||
ONNXTensorElementDataType,
|
||||
size_t, size_t)>;
|
||||
|
||||
public:
|
||||
// Uses the onnxruntime to load the model
|
||||
// into a session.
|
||||
//
|
||||
OnnxPrediction(std::wstring& onnx_model_file);
|
||||
OnnxPrediction(std::wstring& onnx_model_file, Ort::Env& env);
|
||||
|
||||
// Uses the onnx model to create a prediction
|
||||
// environment
|
||||
//
|
||||
OnnxPrediction(onnx::ModelProto& onnx_model);
|
||||
OnnxPrediction(onnx::ModelProto& onnx_model, Ort::Env& env);
|
||||
|
||||
// Deletes the prediction object
|
||||
// The following constructor is meant for initializing using flatbuffer model.
|
||||
// Memory buffer pointing to the model
|
||||
//
|
||||
~OnnxPrediction();
|
||||
OnnxPrediction(const std::vector<char>& model_data, Ort::Env& env);
|
||||
|
||||
// Data to run prediction on
|
||||
//
|
||||
template<typename T>
|
||||
void operator<<(std::vector<T>&& raw_data)
|
||||
{
|
||||
if( currInputIndex >= ptrSession->GetInputCount())
|
||||
{
|
||||
template <typename T>
|
||||
void operator<<(std::vector<T>&& raw_data) {
|
||||
if (curr_input_index >= ptr_session->GetInputCount()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Ort::Value& inputValue = inputValues[currInputIndex];
|
||||
Ort::Value& input_value = input_values[curr_input_index];
|
||||
auto data_size_in_bytes = raw_data.size() * sizeof(T);
|
||||
|
||||
// TODO The following allocation may be unnecessary
|
||||
// Copy the raw input data and control the lifetime.
|
||||
//
|
||||
inputData.emplace_back(alloc.Alloc(data_size_in_bytes),
|
||||
[this](void * ptr1)
|
||||
{
|
||||
this->GetAllocator().Free(ptr1);
|
||||
}
|
||||
);
|
||||
|
||||
std::copy(raw_data.begin(), raw_data.end(), reinterpret_cast<T*>(inputData[currInputIndex].get()));
|
||||
auto inputType = ptrSession->GetInputTypeInfo(currInputIndex);
|
||||
auto shapeInfo = inputType.GetTensorTypeAndShapeInfo().GetShape();
|
||||
auto elem_type = inputType.GetTensorTypeAndShapeInfo().GetElementType();
|
||||
if ( elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
|
||||
{
|
||||
inputValue = Ort::Value::CreateTensor(alloc.GetInfo(),
|
||||
inputData[currInputIndex].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type);
|
||||
}
|
||||
else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32)
|
||||
{
|
||||
inputValue = Ort::Value::CreateTensor(alloc.GetInfo(),
|
||||
inputData[currInputIndex].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type);
|
||||
}
|
||||
else
|
||||
{
|
||||
input_data.emplace_back(alloc.Alloc(data_size_in_bytes),
|
||||
[this](void* ptr1) {
|
||||
this->GetAllocator().Free(ptr1);
|
||||
});
|
||||
|
||||
std::copy(raw_data.begin(), raw_data.end(), reinterpret_cast<T*>(input_data[curr_input_index].get()));
|
||||
auto input_type = ptr_session->GetInputTypeInfo(curr_input_index);
|
||||
auto shapeInfo = input_type.GetTensorTypeAndShapeInfo().GetShape();
|
||||
auto elem_type = input_type.GetTensorTypeAndShapeInfo().GetElementType();
|
||||
if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
|
||||
input_value = Ort::Value::CreateTensor(alloc.GetInfo(),
|
||||
input_data[curr_input_index].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type);
|
||||
} else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
input_value = Ort::Value::CreateTensor(alloc.GetInfo(),
|
||||
input_data[curr_input_index].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type);
|
||||
} else {
|
||||
throw std::exception("only floats are implemented");
|
||||
}
|
||||
|
||||
// Insert data into the next input type
|
||||
//
|
||||
currInputIndex++;
|
||||
curr_input_index++;
|
||||
}
|
||||
|
||||
// Used to generate
|
||||
// Used to generate
|
||||
//
|
||||
void SetupInput(
|
||||
InputGeneratorFunctionType GenerateData,
|
||||
size_t seed = static_cast<unsigned int>(std::chrono::system_clock::now().time_since_epoch().count()));
|
||||
InputGeneratorFunctionType GenerateData,
|
||||
size_t seed = static_cast<unsigned int>(std::chrono::system_clock::now().time_since_epoch().count()));
|
||||
|
||||
// Run the prediction
|
||||
//
|
||||
|
|
@ -108,20 +100,13 @@ public:
|
|||
|
||||
// Do operation on the output data
|
||||
//
|
||||
template<typename T>
|
||||
void ProcessOutputData(T process_function, Ort::Value& val)
|
||||
{
|
||||
if ( val.IsTensor() )
|
||||
{
|
||||
if (val.GetTensorTypeAndShapeInfo().GetElementType()
|
||||
== ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
|
||||
{
|
||||
template <typename T>
|
||||
void ProcessOutputData(T process_function, Ort::Value& val) {
|
||||
if (val.IsTensor()) {
|
||||
if (val.GetTensorTypeAndShapeInfo().GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
|
||||
auto ptr = val.GetTensorMutableData<float>();
|
||||
process_function(ptr, val);
|
||||
}
|
||||
else if (val.GetTensorTypeAndShapeInfo().GetElementType()
|
||||
== ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32)
|
||||
{
|
||||
} else if (val.GetTensorTypeAndShapeInfo().GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
auto ptr = val.GetTensorMutableData<int32_t>();
|
||||
process_function(ptr, val);
|
||||
}
|
||||
|
|
@ -132,7 +117,7 @@ public:
|
|||
//
|
||||
void PrintOutputValues();
|
||||
|
||||
private:
|
||||
private:
|
||||
// Common initilization amongst constructors
|
||||
//
|
||||
void init();
|
||||
|
|
@ -141,59 +126,51 @@ private:
|
|||
//
|
||||
Ort::AllocatorWithDefaultOptions& GetAllocator();
|
||||
|
||||
private:
|
||||
private:
|
||||
// Create an allocator for the runtime to use
|
||||
//
|
||||
Ort::AllocatorWithDefaultOptions alloc;
|
||||
|
||||
// Create Environment
|
||||
//
|
||||
Ort::Env env;
|
||||
|
||||
// Create Options for the Session
|
||||
//
|
||||
Ort::SessionOptions emptySessionOption;
|
||||
Ort::SessionOptions empty_session_option;
|
||||
|
||||
// A pointer to the model
|
||||
//
|
||||
std::shared_ptr<void> rawModel;
|
||||
std::shared_ptr<void> raw_model;
|
||||
|
||||
// Create RunOptions
|
||||
//
|
||||
Ort::RunOptions runOptions;
|
||||
|
||||
// Create a Session to run
|
||||
//
|
||||
Ort::Session session;
|
||||
Ort::RunOptions run_options;
|
||||
|
||||
// Pointer to the current session object
|
||||
//
|
||||
std::unique_ptr<Ort::Session> ptrSession;
|
||||
std::unique_ptr<Ort::Session> ptr_session;
|
||||
|
||||
// Create a list of input values
|
||||
//
|
||||
std::vector<Ort::Value> inputValues{};
|
||||
std::vector<Ort::Value> input_values{};
|
||||
|
||||
// Stores the input names
|
||||
//
|
||||
std::vector<char *> inputNames;
|
||||
std::vector<char*> input_names;
|
||||
|
||||
// Stores the output names
|
||||
//
|
||||
std::vector<char *> outputNames;
|
||||
std::vector<char*> output_names;
|
||||
|
||||
// Create a list of output values
|
||||
//
|
||||
std::vector<Ort::Value> outputValues{};
|
||||
std::vector<Ort::Value> output_values{};
|
||||
|
||||
// We own the lifetime of the input data
|
||||
//
|
||||
std::vector<std::shared_ptr<void>> inputData;
|
||||
std::vector<std::shared_ptr<void>> input_data;
|
||||
|
||||
// Keeps track of number of columns/dimensions data
|
||||
// given for predicition
|
||||
//
|
||||
size_t currInputIndex{0};
|
||||
size_t curr_input_index{0};
|
||||
};
|
||||
|
||||
// OnnxPrediction console output format
|
||||
|
|
@ -204,6 +181,6 @@ std::ostream& operator<<(std::ostream& out, OnnxPrediction& pred);
|
|||
// Used to Generate data for predict
|
||||
//
|
||||
void GenerateDataForInputTypeTensor(OnnxPrediction& predict,
|
||||
size_t input_index, const std::string& input_name,
|
||||
ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed);
|
||||
size_t input_index, const std::string& input_name,
|
||||
ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -2,144 +2,125 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include "OnnxPrediction.h"
|
||||
#include "onnxruntime_session_options_config_keys.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
// Uses the onnxruntime to load the model
|
||||
// into a session.
|
||||
//
|
||||
OnnxPrediction::OnnxPrediction(std::wstring& onnx_model_file)
|
||||
:
|
||||
rawModel{nullptr},
|
||||
ptrSession{nullptr},
|
||||
session{ env, onnx_model_file.c_str(), emptySessionOption},
|
||||
inputNames{session.GetInputCount()},
|
||||
outputNames{session.GetOutputCount()}
|
||||
{
|
||||
OnnxPrediction::OnnxPrediction(std::wstring& onnx_model_file, Ort::Env& env)
|
||||
: raw_model{nullptr},
|
||||
ptr_session{std::make_unique<Ort::Session>(env, onnx_model_file.c_str(), empty_session_option)},
|
||||
input_names(ptr_session->GetInputCount()),
|
||||
output_names(ptr_session->GetOutputCount()) {
|
||||
init();
|
||||
}
|
||||
|
||||
// Uses the onnx to seri
|
||||
//
|
||||
OnnxPrediction::OnnxPrediction(onnx::ModelProto& onnx_model)
|
||||
:
|
||||
session{nullptr}
|
||||
{
|
||||
rawModel = std::shared_ptr<void>{alloc.Alloc(onnx_model.ByteSizeLong()),
|
||||
[this](void * ptr)
|
||||
{
|
||||
this->GetAllocator().Free(ptr);
|
||||
}
|
||||
};
|
||||
OnnxPrediction::OnnxPrediction(onnx::ModelProto& onnx_model, Ort::Env& env) {
|
||||
raw_model = std::shared_ptr<void>(alloc.Alloc(onnx_model.ByteSizeLong()), [this](void* ptr) {
|
||||
this->GetAllocator().Free(ptr);
|
||||
});
|
||||
|
||||
onnx_model.SerializeToArray(rawModel.get(), static_cast<int>(onnx_model.ByteSizeLong()));
|
||||
onnx_model.SerializeToArray(raw_model.get(), static_cast<int>(onnx_model.ByteSizeLong()));
|
||||
|
||||
ptrSession = std::make_unique<Ort::Session>(env,
|
||||
rawModel.get(),
|
||||
onnx_model.ByteSizeLong(),
|
||||
emptySessionOption),
|
||||
ptr_session = std::make_unique<Ort::Session>(env,
|
||||
static_cast<void*>(raw_model.get()),
|
||||
onnx_model.ByteSizeLong(),
|
||||
empty_session_option);
|
||||
|
||||
inputNames.resize(ptrSession->GetInputCount());
|
||||
outputNames.resize(ptrSession->GetOutputCount());
|
||||
// TODO Use reserve instead of resize
|
||||
input_names.resize(ptr_session->GetInputCount());
|
||||
output_names.resize(ptr_session->GetOutputCount());
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
// Destructor
|
||||
//
|
||||
OnnxPrediction::~OnnxPrediction()
|
||||
{
|
||||
if (ptrSession.get() == &session)
|
||||
{
|
||||
// Ensure the session is not deleted
|
||||
// by the unique_ptr. Because it will be deleting the stack
|
||||
//
|
||||
ptrSession.release();
|
||||
}
|
||||
OnnxPrediction::OnnxPrediction(const std::vector<char>& model_data, Ort::Env& env) {
|
||||
Ort::SessionOptions so;
|
||||
so.AddConfigEntry(kOrtSessionOptionsConfigLoadModelFormat, "ORT");
|
||||
ptr_session = std::make_unique<Ort::Session>(env,
|
||||
model_data.data(),
|
||||
model_data.size(),
|
||||
so);
|
||||
|
||||
// TODO Use reserve instead of resize
|
||||
input_names.resize(ptr_session->GetInputCount());
|
||||
output_names.resize(ptr_session->GetOutputCount());
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
// OnnxPrediction console output format
|
||||
// prints the output data.
|
||||
//
|
||||
std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred)
|
||||
{
|
||||
auto pretty_print = [&out](auto ptr, Ort::Value& val)
|
||||
{
|
||||
out << L"[";
|
||||
std::wstring msg = L"";
|
||||
for(int i=0; i < val.GetTensorTypeAndShapeInfo().GetElementCount(); i++)
|
||||
{
|
||||
out << msg << ptr[i];
|
||||
msg = L", ";
|
||||
}
|
||||
out << L"]\n";
|
||||
std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred) {
|
||||
auto pretty_print = [&out](auto ptr, Ort::Value& val) {
|
||||
out << L"[";
|
||||
std::wstring msg = L"";
|
||||
for (int i = 0; i < val.GetTensorTypeAndShapeInfo().GetElementCount(); i++) {
|
||||
out << msg << ptr[i];
|
||||
msg = L", ";
|
||||
}
|
||||
out << L"]\n";
|
||||
};
|
||||
|
||||
size_t index {0};
|
||||
for(auto& val : pred.outputValues)
|
||||
{
|
||||
out << pred.outputNames[index++] << L" = ";
|
||||
size_t index{0};
|
||||
for (auto& val : pred.output_values) {
|
||||
out << pred.output_names[index++] << L" = ";
|
||||
pred.ProcessOutputData(pretty_print, val);
|
||||
}
|
||||
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// Used to Generate data for predict
|
||||
//
|
||||
void GenerateDataForInputTypeTensor(OnnxPrediction& predict,
|
||||
size_t input_index, const std::string& input_name,
|
||||
ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed)
|
||||
{
|
||||
(void) input_name;
|
||||
(void) input_index;
|
||||
|
||||
auto pretty_print = [&input_name](auto raw_data)
|
||||
{
|
||||
size_t input_index, const std::string& input_name,
|
||||
ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed) {
|
||||
(void)input_name;
|
||||
(void)input_index;
|
||||
|
||||
auto pretty_print = [&input_name](auto raw_data) {
|
||||
Logger::testLog << input_name << L" = ";
|
||||
Logger::testLog << L"[";
|
||||
std::wstring msg = L"";
|
||||
for(int i=0; i < raw_data.size(); i++)
|
||||
{
|
||||
for (int i = 0; i < raw_data.size(); i++) {
|
||||
Logger::testLog << msg << raw_data[i];
|
||||
msg = L", ";
|
||||
}
|
||||
Logger::testLog << L"]\n";
|
||||
Logger::testLog << L"]\n";
|
||||
};
|
||||
|
||||
if ( elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)
|
||||
{
|
||||
auto raw_data = GenerateRandomData(0.0f, elem_count,seed);
|
||||
pretty_print(raw_data);
|
||||
if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) {
|
||||
auto raw_data = GenerateRandomData(0.0f, elem_count, seed);
|
||||
// pretty_print(raw_data);
|
||||
predict << std::move(raw_data);
|
||||
}
|
||||
else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32)
|
||||
{
|
||||
} else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) {
|
||||
int32_t initial = 0;
|
||||
auto raw_data = GenerateRandomData(initial, elem_count,seed);
|
||||
pretty_print(raw_data);
|
||||
auto raw_data = GenerateRandomData(initial, elem_count, seed);
|
||||
// pretty_print(raw_data);
|
||||
predict << std::move(raw_data);
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
throw std::exception("only floats are implemented");
|
||||
}
|
||||
}
|
||||
|
||||
// Run the prediction
|
||||
//
|
||||
void OnnxPrediction::RunInference()
|
||||
{
|
||||
void OnnxPrediction::RunInference() {
|
||||
Logger::testLog << L"inference starting " << Logger::endl;
|
||||
Logger::testLog.flush();
|
||||
|
||||
try
|
||||
{
|
||||
ptrSession->Run(runOptions,
|
||||
inputNames.data(), inputValues.data(),
|
||||
inputValues.size(), outputNames.data(), outputValues.data(),
|
||||
outputNames.size());
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
try {
|
||||
ptr_session->Run(run_options,
|
||||
input_names.data(), input_values.data(),
|
||||
input_values.size(), output_names.data(), output_values.data(),
|
||||
output_names.size());
|
||||
} catch (...) {
|
||||
Logger::testLog << L"Something went wrong in inference " << Logger::endl;
|
||||
Logger::testLog.flush();
|
||||
throw;
|
||||
|
|
@ -151,8 +132,7 @@ void OnnxPrediction::RunInference()
|
|||
|
||||
// Print the output values of the prediction.
|
||||
//
|
||||
void OnnxPrediction::PrintOutputValues()
|
||||
{
|
||||
void OnnxPrediction::PrintOutputValues() {
|
||||
Logger::testLog << L"output data:\n";
|
||||
Logger::testLog << *this;
|
||||
Logger::testLog << Logger::endl;
|
||||
|
|
@ -160,68 +140,49 @@ void OnnxPrediction::PrintOutputValues()
|
|||
|
||||
// Common initilization amongst constructors
|
||||
//
|
||||
void OnnxPrediction::init()
|
||||
{
|
||||
// Enable telemetry events
|
||||
//
|
||||
env.EnableTelemetryEvents();
|
||||
|
||||
if (!ptrSession)
|
||||
{
|
||||
// To use one consistent value
|
||||
// across the class
|
||||
//
|
||||
ptrSession.reset(&session);
|
||||
}
|
||||
|
||||
void OnnxPrediction::init() {
|
||||
// Initialize model input names
|
||||
//
|
||||
for(int i=0; i < ptrSession->GetInputCount(); i++)
|
||||
{
|
||||
inputNames[i] = ptrSession->GetInputName(i, alloc);
|
||||
inputValues.emplace_back(nullptr);
|
||||
for (int i = 0; i < ptr_session->GetInputCount(); i++) {
|
||||
// TODO Use push_back on input_names instead of assignment
|
||||
input_names[i] = ptr_session->GetInputName(i, alloc);
|
||||
input_values.emplace_back(nullptr);
|
||||
}
|
||||
|
||||
// Initialize model output names
|
||||
//
|
||||
for(int i=0; i < ptrSession->GetOutputCount(); i++)
|
||||
{
|
||||
outputNames[i] = ptrSession->GetOutputName(i, alloc);
|
||||
outputValues.emplace_back(nullptr);
|
||||
for (int i = 0; i < ptr_session->GetOutputCount(); i++) {
|
||||
// TODO Use push_back on output_names instead of assignment
|
||||
output_names[i] = ptr_session->GetOutputName(i, alloc);
|
||||
output_values.emplace_back(nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
// Get the allocator used by the runtime
|
||||
//
|
||||
Ort::AllocatorWithDefaultOptions& OnnxPrediction::GetAllocator()
|
||||
{
|
||||
Ort::AllocatorWithDefaultOptions& OnnxPrediction::GetAllocator() {
|
||||
return alloc;
|
||||
}
|
||||
|
||||
void OnnxPrediction::SetupInput(
|
||||
InputGeneratorFunctionType GenerateData,
|
||||
size_t seed)
|
||||
{
|
||||
InputGeneratorFunctionType GenerateData,
|
||||
size_t seed) {
|
||||
Logger::testLog << L"input data:\n";
|
||||
for(int i=0; i < ptrSession->GetInputCount(); i++)
|
||||
{
|
||||
auto inputType = ptrSession->GetInputTypeInfo(i);
|
||||
for (int i = 0; i < ptr_session->GetInputCount(); i++) {
|
||||
auto inputType = ptr_session->GetInputTypeInfo(i);
|
||||
|
||||
if (inputType.GetONNXType() == ONNX_TYPE_TENSOR)
|
||||
{
|
||||
if (inputType.GetONNXType() == ONNX_TYPE_TENSOR) {
|
||||
auto elem_type = inputType.GetTensorTypeAndShapeInfo().GetElementType();
|
||||
auto elem_count = inputType.GetTensorTypeAndShapeInfo().GetElementCount();
|
||||
|
||||
|
||||
// This can be any generic function to generate inputs
|
||||
//
|
||||
GenerateData(*this, i , std::string(inputNames[i]), elem_type, elem_count, seed);
|
||||
GenerateData(*this, i, std::string(input_names[i]), elem_type, elem_count, seed);
|
||||
|
||||
// Update the seed in a predicatable way to get other values for different inputs
|
||||
//
|
||||
seed++;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
std::cout << "Unsupported \n";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,191 +4,176 @@
|
|||
#include "src/mutator.h"
|
||||
#include "testlog.h"
|
||||
#include "OnnxPrediction.h"
|
||||
#include "onnxruntime_session_options_config_keys.h"
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
using userOptions = struct
|
||||
using user_options = struct
|
||||
{
|
||||
bool writeModel;
|
||||
bool write_model;
|
||||
bool verbose;
|
||||
bool stress;
|
||||
bool is_ort;
|
||||
};
|
||||
|
||||
void predict(onnx::ModelProto& model_proto, unsigned int seed)
|
||||
{
|
||||
// Create object for prediction
|
||||
//
|
||||
OnnxPrediction predict{model_proto};
|
||||
void predict(onnx::ModelProto& model_proto, unsigned int seed, Ort::Env& env) {
|
||||
// Create object for prediction
|
||||
//
|
||||
OnnxPrediction predict(model_proto, env);
|
||||
|
||||
// Give predict a function to generate the data
|
||||
// to run prediction on.
|
||||
//
|
||||
predict.SetupInput(GenerateDataForInputTypeTensor, seed);
|
||||
// Give predict a function to generate the data
|
||||
// to run prediction on.
|
||||
//
|
||||
predict.SetupInput(GenerateDataForInputTypeTensor, seed);
|
||||
|
||||
// Run the prediction on the data
|
||||
//
|
||||
predict.RunInference();
|
||||
// Run the prediction on the data
|
||||
//
|
||||
predict.RunInference();
|
||||
|
||||
// View the output
|
||||
//
|
||||
predict.PrintOutputValues();
|
||||
// View the output
|
||||
//
|
||||
predict.PrintOutputValues();
|
||||
}
|
||||
|
||||
void mutateModelTest(onnx::ModelProto& model_proto,
|
||||
std::wstring mutatedModelDirName,
|
||||
userOptions opt,
|
||||
unsigned int seed = 0)
|
||||
{
|
||||
// Used to initialize all random engines
|
||||
void mutateModelTest(onnx::ModelProto& model_proto,
|
||||
std::wstring mutatedModelDirName,
|
||||
user_options opt,
|
||||
Ort::Env& env,
|
||||
unsigned int seed = 0) {
|
||||
// Used to initialize all random engines
|
||||
//
|
||||
std::wstring modelPrefix = L"/ReproMutateModel_";
|
||||
if (seed == 0) {
|
||||
seed = static_cast<unsigned int>(std::chrono::system_clock::now().time_since_epoch().count());
|
||||
modelPrefix = L"/MutateModel_";
|
||||
}
|
||||
|
||||
if (opt.stress) {
|
||||
Logger::testLog.enable();
|
||||
}
|
||||
|
||||
Logger::testLog << L"Mutate test seed: " << seed << Logger::endl;
|
||||
opt.stress ? Logger::testLog.disable() : Logger::testLog.enable();
|
||||
|
||||
// Create mutator
|
||||
//
|
||||
protobuf_mutator::Mutator mutator;
|
||||
|
||||
// Mutate model
|
||||
//
|
||||
Logger::testLog << L"Model Successfully Initialized" << Logger::endl;
|
||||
mutator.Seed(seed);
|
||||
mutator.Mutate(&model_proto, model_proto.ByteSizeLong());
|
||||
|
||||
if (opt.write_model) {
|
||||
// Create file to store model
|
||||
//
|
||||
std::wstring modelPrefix = L"/ReproMutateModel_";
|
||||
if(seed == 0)
|
||||
{
|
||||
seed = static_cast<unsigned int>(std::chrono::system_clock::now().time_since_epoch().count());
|
||||
modelPrefix = L"/MutateModel_";
|
||||
}
|
||||
|
||||
if(opt.stress)
|
||||
{
|
||||
Logger::testLog.enable();
|
||||
}
|
||||
std::wstringstream mutateModelName;
|
||||
mutateModelName << mutatedModelDirName << modelPrefix << seed << L".onnx";
|
||||
auto mutateModelFileName = mutateModelName.str();
|
||||
|
||||
Logger::testLog << L"Mutate test seed: " << seed << Logger::endl;
|
||||
opt.stress ? Logger::testLog.disable() : Logger::testLog.enable();
|
||||
|
||||
// Create mutator
|
||||
// Log the model to a file
|
||||
//
|
||||
protobuf_mutator::Mutator mutator;
|
||||
|
||||
// Mutate model
|
||||
std::ofstream outStream(mutateModelFileName);
|
||||
model_proto.SerializeToOstream(&outStream);
|
||||
Logger::testLog << "Mutated Model Written to file: " << mutateModelFileName << Logger::endl;
|
||||
|
||||
// Flush the buffer to ensure the
|
||||
// mutated model info for reproduction
|
||||
// purposes.
|
||||
//
|
||||
Logger::testLog << L"Model Successfully Initialized" << Logger::endl;
|
||||
mutator.Seed(seed);
|
||||
mutator.Mutate(&model_proto, model_proto.ByteSizeLong());
|
||||
outStream << std::flush;
|
||||
}
|
||||
|
||||
if (opt.writeModel)
|
||||
{
|
||||
// Create file to store model
|
||||
//
|
||||
std::wstringstream mutateModelName;
|
||||
mutateModelName << mutatedModelDirName << modelPrefix << seed << L".onnx";
|
||||
auto mutateModelFileName = mutateModelName.str();
|
||||
// Flush any logs before prediction
|
||||
//
|
||||
Logger::testLog.flush();
|
||||
|
||||
// Log the model to a file
|
||||
//
|
||||
std::ofstream outStream(mutateModelFileName);
|
||||
model_proto.SerializeToOstream(&outStream);
|
||||
Logger::testLog<< "Mutated Model Written to file: " << mutateModelFileName << Logger::endl;
|
||||
// run prediction on model
|
||||
//
|
||||
predict(model_proto, seed, env);
|
||||
|
||||
// Flush the buffer to ensure the
|
||||
// mutated model info for reproduction
|
||||
// purposes.
|
||||
//
|
||||
outStream << std::flush;
|
||||
}
|
||||
|
||||
// Flush any logs before prediction
|
||||
//
|
||||
Logger::testLog.flush();
|
||||
|
||||
// run prediction on model
|
||||
//
|
||||
predict(model_proto, seed);
|
||||
|
||||
// print out all output before next test
|
||||
//
|
||||
Logger::testLog.flush();
|
||||
// print out all output before next test
|
||||
//
|
||||
Logger::testLog.flush();
|
||||
}
|
||||
|
||||
void printUsage()
|
||||
{
|
||||
void printUsage() {
|
||||
std::cout << "Not enough command line arguments\n";
|
||||
std::cout << "usage:\n"
|
||||
<< "\t\tFor testing: test.exe /t [options] onnx_model_file test_timeout test_time_scale\n"
|
||||
<< "\t\tFor repro/debugging: test.exe /r onnx_model_file seed\n"
|
||||
<< "\n\nonnx_model_file: Unmutated onnx model file\n"
|
||||
<< "options: /m - output mutated models /v - verbose logging /s - stress test"
|
||||
<< "test_time_scale: h|m|s\n"
|
||||
<< "test_timeout: Time to run the test in hrs\n"
|
||||
<< "seed: The seed that generated the mutated model. This value is the decimal digit part of the mutated model name (or can be found in the logs)\n"
|
||||
<< "\n";
|
||||
<< "\t\tFor testing: test.exe /t [options] onnx_model_file test_timeout test_time_scale\n"
|
||||
<< "\t\tFor repro/debugging: test.exe /r onnx_model_file seed\n"
|
||||
<< "\n\nonnx_model_file: Unmutated onnx model file\n"
|
||||
<< "options: /m - output mutated models /v - verbose logging /s - stress test"
|
||||
<< "test_time_scale: h|m|s\n"
|
||||
<< "test_timeout: Time to run the test in hrs\n"
|
||||
<< "seed: The seed that generated the mutated model. This value is the decimal digit part of the mutated model name (or can be found in the logs)\n"
|
||||
<< "\n";
|
||||
}
|
||||
|
||||
enum class timeScale : char
|
||||
{
|
||||
Hrs = 'h',
|
||||
enum class timeScale : char {
|
||||
Hrs = 'h',
|
||||
Min = 'm',
|
||||
Sec = 's'
|
||||
};
|
||||
|
||||
using runtimeOpt = struct
|
||||
{
|
||||
std::wstring modelFileName{};
|
||||
std::wstring mutateModelDirName{};
|
||||
Logger::ccstream errStreamBuf{};
|
||||
Logger::wcstream werrStreamBuf{};
|
||||
bool repoMode{false};
|
||||
int testTimeOut{0};
|
||||
struct runtimeOpt {
|
||||
std::wstring model_file_name{};
|
||||
std::wstring mutate_model_dir_name{};
|
||||
Logger::ccstream err_stream_buf{};
|
||||
Logger::wcstream werr_stream_buf{};
|
||||
bool repo_mode{false};
|
||||
int test_time_out{0};
|
||||
unsigned int seed{0};
|
||||
timeScale scale{timeScale::Sec};
|
||||
userOptions userOpt{false, false, false};
|
||||
user_options user_opt{false, false, false, false};
|
||||
};
|
||||
|
||||
int processCommandLine(int argc, char* argv[], runtimeOpt& opt)
|
||||
{
|
||||
if (argc <= 1)
|
||||
{
|
||||
int processCommandLine(int argc, char* argv[], runtimeOpt& opt) {
|
||||
if (argc <= 1) {
|
||||
printUsage();
|
||||
return 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
bool isTest = std::string{argv[1]} == "/t";
|
||||
bool isRepo = std::string{argv[1]} == "/r";
|
||||
|
||||
if(isRepo)
|
||||
{
|
||||
opt.repoMode = true;
|
||||
opt.mutateModelDirName = L"./repromodel";
|
||||
std::filesystem::path mutateModelDir{opt.mutateModelDirName};
|
||||
if ( !std::filesystem::exists(mutateModelDir) )
|
||||
{
|
||||
std::filesystem::create_directory(mutateModelDir);
|
||||
if (isRepo) {
|
||||
opt.repo_mode = true;
|
||||
opt.mutate_model_dir_name = L"./repromodel";
|
||||
std::filesystem::path mutate_model_dir{opt.mutate_model_dir_name};
|
||||
if (!std::filesystem::exists(mutate_model_dir)) {
|
||||
std::filesystem::create_directory(mutate_model_dir);
|
||||
}
|
||||
|
||||
opt.modelFileName = Logger::towstr(argv[2]);
|
||||
Logger::testLog<< L"Repo Model file: " << opt.modelFileName << Logger::endl;
|
||||
opt.model_file_name = Logger::towstr(argv[2]);
|
||||
Logger::testLog << L"Repo Model file: " << opt.model_file_name << Logger::endl;
|
||||
|
||||
// Get seed
|
||||
//
|
||||
std::stringstream parser{argv[3]};
|
||||
parser >> opt.seed;
|
||||
if (parser.bad())
|
||||
{
|
||||
if (parser.bad()) {
|
||||
throw std::exception("Could not parse seed from command line");
|
||||
}
|
||||
|
||||
std::wcout << L"seed: " << opt.seed << L"\n";
|
||||
}
|
||||
else if (isTest)
|
||||
{
|
||||
std::wcout << L"seed: " << opt.seed << L"\n";
|
||||
} else if (isTest) {
|
||||
int index{argc};
|
||||
index--;
|
||||
|
||||
// Parse right to left
|
||||
//
|
||||
std::stringstream parser;
|
||||
char desiredScale;
|
||||
char desired_scale;
|
||||
parser << argv[index--];
|
||||
parser >> desiredScale;
|
||||
|
||||
if (parser.bad())
|
||||
{
|
||||
parser >> desired_scale;
|
||||
|
||||
if (parser.bad()) {
|
||||
throw std::exception("Could not parse the time scale from the command line");
|
||||
}
|
||||
|
||||
opt.scale = static_cast<timeScale>(std::tolower(desiredScale));
|
||||
switch(opt.scale)
|
||||
{
|
||||
|
||||
opt.scale = static_cast<timeScale>(std::tolower(desired_scale));
|
||||
switch (opt.scale) {
|
||||
case timeScale::Hrs:
|
||||
case timeScale::Min:
|
||||
case timeScale::Sec:
|
||||
|
|
@ -196,67 +181,56 @@ int processCommandLine(int argc, char* argv[], runtimeOpt& opt)
|
|||
default:
|
||||
throw std::exception("Could not parse the time scale from the command line");
|
||||
}
|
||||
|
||||
|
||||
parser << argv[index--];
|
||||
parser >> opt.testTimeOut;
|
||||
if (parser.bad())
|
||||
{
|
||||
parser >> opt.test_time_out;
|
||||
if (parser.bad()) {
|
||||
throw std::exception("Could not parse the time value from the command line");
|
||||
}
|
||||
|
||||
Logger::testLog<< L"Running Test for: " << opt.testTimeOut << desiredScale << Logger::endl;
|
||||
opt.modelFileName = Logger::towstr(argv[index--]);
|
||||
Logger::testLog<< L"Model file: " << opt.modelFileName << Logger::endl;
|
||||
std::filesystem::path modelFileNamePath{opt.modelFileName};
|
||||
if (!std::filesystem::exists(modelFileNamePath))
|
||||
{
|
||||
Logger::testLog << L"Running Test for: " << opt.test_time_out << desired_scale << Logger::endl;
|
||||
opt.model_file_name = Logger::towstr(argv[index--]);
|
||||
Logger::testLog << L"Model file: " << opt.model_file_name << Logger::endl;
|
||||
std::filesystem::path model_file_namePath{opt.model_file_name};
|
||||
if (!std::filesystem::exists(model_file_namePath)) {
|
||||
throw std::exception("Cannot find model file");
|
||||
}
|
||||
|
||||
// process options
|
||||
//
|
||||
while(index > 0)
|
||||
{
|
||||
while (index > 0) {
|
||||
auto option{std::string{argv[index]}};
|
||||
if ( option == "/m")
|
||||
{
|
||||
opt.userOpt.writeModel = true;
|
||||
}
|
||||
else if (option == "/v")
|
||||
{
|
||||
opt.userOpt.verbose = true;
|
||||
}
|
||||
else if (option == "/s")
|
||||
{
|
||||
opt.userOpt.stress = true;
|
||||
if (option == "/m") {
|
||||
opt.user_opt.write_model = true;
|
||||
} else if (option == "/v") {
|
||||
opt.user_opt.verbose = true;
|
||||
} else if (option == "/s") {
|
||||
opt.user_opt.stress = true;
|
||||
} else if (option == "/f") {
|
||||
opt.user_opt.is_ort = true;
|
||||
}
|
||||
index--;
|
||||
}
|
||||
|
||||
if (opt.userOpt.stress)
|
||||
{
|
||||
std::cerr.rdbuf(&opt.errStreamBuf);
|
||||
std::wcerr.rdbuf(&opt.werrStreamBuf);
|
||||
opt.userOpt.writeModel = false;
|
||||
opt.userOpt.verbose = false;
|
||||
if (opt.user_opt.stress) {
|
||||
std::cerr.rdbuf(&opt.err_stream_buf);
|
||||
std::wcerr.rdbuf(&opt.werr_stream_buf);
|
||||
opt.user_opt.write_model = false;
|
||||
opt.user_opt.verbose = false;
|
||||
Logger::testLog.disable();
|
||||
Logger::testLog.minLog();
|
||||
}
|
||||
|
||||
// create directory for mutated model output
|
||||
// create directory for mutated model output
|
||||
//
|
||||
if(opt.userOpt.writeModel)
|
||||
{
|
||||
opt.mutateModelDirName = L"./mutatemodel";
|
||||
std::filesystem::path mutateModelDir{opt.mutateModelDirName};
|
||||
if ( !std::filesystem::exists(mutateModelDir) )
|
||||
{
|
||||
std::filesystem::create_directory(mutateModelDir);
|
||||
if (opt.user_opt.write_model) {
|
||||
opt.mutate_model_dir_name = L"./mutatemodel";
|
||||
std::filesystem::path mutate_model_dir{opt.mutate_model_dir_name};
|
||||
if (!std::filesystem::exists(mutate_model_dir)) {
|
||||
std::filesystem::create_directory(mutate_model_dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
printUsage();
|
||||
return 2;
|
||||
}
|
||||
|
|
@ -265,143 +239,170 @@ int processCommandLine(int argc, char* argv[], runtimeOpt& opt)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
struct RunStats {
|
||||
size_t num_ort_exception;
|
||||
size_t num_std_exception;
|
||||
size_t num_unknown_exception;
|
||||
size_t num_successful_runs;
|
||||
size_t iteration;
|
||||
};
|
||||
|
||||
static void fuzz_handle_exception(struct RunStats& run_stats) {
|
||||
try {
|
||||
throw;
|
||||
} catch (const Ort::Exception& ortException) {
|
||||
run_stats.num_ort_exception++;
|
||||
Logger::testLog << L"onnx runtime exception: " << ortException.what() << Logger::endl;
|
||||
Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl;
|
||||
} catch (const std::exception& e) {
|
||||
run_stats.num_std_exception++;
|
||||
Logger::testLog << L"standard exception: " << e.what() << Logger::endl;
|
||||
Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl;
|
||||
} catch (...) {
|
||||
run_stats.num_unknown_exception++;
|
||||
Logger::testLog << L"unknown exception: " << Logger::endl;
|
||||
Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
Ort::Env env;
|
||||
// Enable telemetry events
|
||||
//
|
||||
env.EnableTelemetryEvents();
|
||||
struct RunStats run_stats = {0, 0, 0, 0, 0};
|
||||
runtimeOpt opt{};
|
||||
try
|
||||
{
|
||||
user_options& user_opt{opt.user_opt};
|
||||
Logger::wcstream& werr_stream_buf{opt.werr_stream_buf};
|
||||
try {
|
||||
// Initialize the runtime options
|
||||
//
|
||||
auto canContinue{processCommandLine(argc, argv, opt) == 0};
|
||||
|
||||
if(!canContinue)
|
||||
{
|
||||
if (!canContinue) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::wstring& modelFileName{opt.modelFileName};
|
||||
std::wstring& mutateModelDirName{opt.mutateModelDirName};
|
||||
bool& repoMode{opt.repoMode};
|
||||
int& testTimeOut{opt.testTimeOut};
|
||||
std::wstring& model_file_name{opt.model_file_name};
|
||||
std::wstring& mutate_model_dir_name{opt.mutate_model_dir_name};
|
||||
bool& repo_mode{opt.repo_mode};
|
||||
int& test_time_out{opt.test_time_out};
|
||||
unsigned int& seed{opt.seed};
|
||||
timeScale& scale{opt.scale};
|
||||
userOptions& userOpt{opt.userOpt};
|
||||
Logger::wcstream& werrStreamBuf{opt.werrStreamBuf};
|
||||
|
||||
|
||||
// Model file
|
||||
//
|
||||
std::wstring modelFile {modelFileName};
|
||||
|
||||
std::wstring model_file{model_file_name};
|
||||
|
||||
// Create a stream to hold the model
|
||||
//
|
||||
std::ifstream modelStream{modelFile};
|
||||
std::ifstream modelStream{model_file, std::ios::in | std::ios::binary};
|
||||
if (opt.user_opt.is_ort == false) {
|
||||
// Create an onnx protobuf object
|
||||
//
|
||||
onnx::ModelProto model_proto;
|
||||
|
||||
// Create an onnx protobuf object
|
||||
//
|
||||
onnx::ModelProto model_proto;
|
||||
|
||||
// Initialize the model
|
||||
//
|
||||
if( model_proto.ParseFromIstream(&modelStream) )
|
||||
{
|
||||
if(repoMode)
|
||||
{
|
||||
Logger::testLog<< L"Running Prediction for: " << modelFileName
|
||||
<< L" with seed " << seed << Logger::endl;
|
||||
mutateModelTest(model_proto, mutateModelDirName, userOpt, seed);
|
||||
Logger::testLog<< L"Finished Prediction for: " << modelFileName
|
||||
<< L" with seed " << seed << Logger::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Call the mutateModelTest
|
||||
//
|
||||
std::chrono::system_clock::time_point currTime{ std::chrono::system_clock::now() };
|
||||
|
||||
std::chrono::minutes timeInMin{testTimeOut};
|
||||
std::chrono::seconds timeInSec{testTimeOut};
|
||||
std::chrono::hours timeInHrs{testTimeOut};
|
||||
std::chrono::system_clock::time_point endTime{currTime};
|
||||
endTime += scale == timeScale::Hrs ? timeInHrs
|
||||
: scale == timeScale::Min ? timeInMin : timeInSec;
|
||||
Logger::testLog<< "Starting Test" << Logger::endl;
|
||||
size_t num_ort_exception{0};
|
||||
size_t num_std_exception{0};
|
||||
size_t num_unknown_exception{0};
|
||||
size_t num_successful_runs{0};
|
||||
size_t iteration{0};
|
||||
while(currTime < endTime)
|
||||
{
|
||||
try
|
||||
{
|
||||
onnx::ModelProto bad_model = model_proto;
|
||||
Logger::testLog<< "Starting Test iteration: " << iteration << Logger::endl;
|
||||
mutateModelTest(bad_model, mutateModelDirName, userOpt);
|
||||
num_successful_runs++;
|
||||
Logger::testLog<< "Completed Test iteration: " << iteration++ << Logger::endl;
|
||||
}
|
||||
catch(const Ort::Exception& ortException)
|
||||
{
|
||||
num_ort_exception++;
|
||||
Logger::testLog<< L"onnx runtime exception: " << ortException.what() << Logger::endl;
|
||||
Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl;
|
||||
}
|
||||
catch(const std::exception& e)
|
||||
{
|
||||
num_std_exception++;
|
||||
Logger::testLog<< L"standard exception: " << e.what() << Logger::endl;
|
||||
Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl;
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
num_unknown_exception++;
|
||||
Logger::testLog<< L"unknown exception: " << Logger::endl;
|
||||
Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl;
|
||||
throw;
|
||||
}
|
||||
|
||||
// Update current time
|
||||
// Initialize the model
|
||||
//
|
||||
if (model_proto.ParseFromIstream(&modelStream)) {
|
||||
if (repo_mode) {
|
||||
Logger::testLog << L"Running Prediction for: " << model_file_name
|
||||
<< L" with seed " << seed << Logger::endl;
|
||||
mutateModelTest(model_proto, mutate_model_dir_name, user_opt, env, seed);
|
||||
Logger::testLog << L"Finished Prediction for: " << model_file_name
|
||||
<< L" with seed " << seed << Logger::endl;
|
||||
return 0;
|
||||
} else {
|
||||
// Call the mutateModelTest
|
||||
//
|
||||
currTime = std::chrono::system_clock::now();
|
||||
}
|
||||
Logger::testLog<< "Ending Test" << Logger::endl;
|
||||
|
||||
if(userOpt.stress)
|
||||
{
|
||||
Logger::testLog.enable();
|
||||
}
|
||||
std::chrono::system_clock::time_point curr_time{std::chrono::system_clock::now()};
|
||||
|
||||
Logger::testLog<< L"Total number of exceptions: " << num_unknown_exception+num_std_exception+num_ort_exception << Logger::endl;
|
||||
Logger::testLog<< L"Number of Unknown exceptions: " << num_unknown_exception << Logger::endl;
|
||||
Logger::testLog<< L"Number of ort exceptions: " << num_ort_exception << Logger::endl;
|
||||
Logger::testLog<< L"Number of std exceptions: " << num_std_exception << Logger::endl;
|
||||
Logger::testLog << L"Number of unique errors: "<< werrStreamBuf.get_unique_errors() << L"\n";
|
||||
|
||||
if(userOpt.stress)
|
||||
{
|
||||
Logger::testLog.disable();
|
||||
Logger::testLog.flush();
|
||||
std::chrono::minutes time_in_min{test_time_out};
|
||||
std::chrono::seconds time_in_sec{test_time_out};
|
||||
std::chrono::hours time_in_hrs{test_time_out};
|
||||
std::chrono::system_clock::time_point end_time{curr_time};
|
||||
end_time += scale == timeScale::Hrs ? time_in_hrs
|
||||
: scale == timeScale::Min ? time_in_min : time_in_sec;
|
||||
Logger::testLog << "Starting Test" << Logger::endl;
|
||||
while (curr_time < end_time) {
|
||||
try {
|
||||
onnx::ModelProto bad_model = model_proto;
|
||||
Logger::testLog << "Starting Test iteration: " << run_stats.iteration << Logger::endl;
|
||||
mutateModelTest(bad_model, mutate_model_dir_name, user_opt, env);
|
||||
run_stats.num_successful_runs++;
|
||||
Logger::testLog << "Completed Test iteration: " << run_stats.iteration++ << Logger::endl;
|
||||
} catch (...) {
|
||||
fuzz_handle_exception(run_stats);
|
||||
}
|
||||
// Update current time
|
||||
//
|
||||
curr_time = std::chrono::system_clock::now();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw std::exception("Unable to initialize the Onnx model in memory");
|
||||
}
|
||||
} else {
|
||||
std::wstring ort_model_file = model_file;
|
||||
if (model_file.substr(model_file.find_last_of(L".") + 1) == L"onnx") {
|
||||
ort_model_file = model_file + L".ort";
|
||||
Ort::SessionOptions so;
|
||||
so.SetGraphOptimizationLevel(ORT_DISABLE_ALL);
|
||||
so.SetOptimizedModelFilePath(ort_model_file.c_str());
|
||||
so.AddConfigEntry(kOrtSessionOptionsConfigSaveModelFormat, "ORT");
|
||||
Ort::Session session(env, model_file.c_str(), so);
|
||||
} else if (model_file.substr(model_file.find_last_of(L".") + 1) != L"ort") {
|
||||
Logger::testLog << L"Input file name extension is not 'onnx' or 'ort' " << Logger::endl;
|
||||
return 1;
|
||||
}
|
||||
size_t num_bytes = std::filesystem::file_size(ort_model_file);
|
||||
std::vector<char> model_data(num_bytes);
|
||||
std::ifstream ortModelStream(ort_model_file, std::ifstream::in | std::ifstream::binary);
|
||||
ortModelStream.read(model_data.data(), num_bytes);
|
||||
ortModelStream.close();
|
||||
// Currently mutations are generated by using XOR of a byte with the preceeding byte at a time.
|
||||
// Other possible ways may be considered in future, for example swaping two bytes randomly at a time.
|
||||
Logger::testLog << "Starting Test" << Logger::endl;
|
||||
for (size_t& i = run_stats.iteration; i < num_bytes - 1; i++) {
|
||||
char tmp = model_data[i];
|
||||
model_data[i] ^= model_data[i + 1];
|
||||
try {
|
||||
Logger::testLog << "Starting Test iteration: " << i << Logger::endl;
|
||||
OnnxPrediction predict(model_data, env);
|
||||
predict.SetupInput(GenerateDataForInputTypeTensor, 0);
|
||||
predict.RunInference();
|
||||
run_stats.num_successful_runs++;
|
||||
Logger::testLog << "Completed Test iteration: " << i << Logger::endl;
|
||||
} catch (...) {
|
||||
fuzz_handle_exception(run_stats);
|
||||
}
|
||||
model_data[i] = tmp;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw std::exception("Unable to initialize the Onnx model in memory");
|
||||
}
|
||||
Logger::testLog << "Ending Test" << Logger::endl;
|
||||
|
||||
if (user_opt.stress) {
|
||||
Logger::testLog.enable();
|
||||
}
|
||||
size_t toal_num_exception = run_stats.num_unknown_exception + run_stats.num_std_exception + run_stats.num_ort_exception;
|
||||
Logger::testLog << L"Total number of exceptions: " << toal_num_exception << Logger::endl;
|
||||
Logger::testLog << L"Number of Unknown exceptions: " << run_stats.num_unknown_exception << Logger::endl;
|
||||
Logger::testLog << L"Number of ort exceptions: " << run_stats.num_ort_exception << Logger::endl;
|
||||
Logger::testLog << L"Number of std exceptions: " << run_stats.num_std_exception << Logger::endl;
|
||||
Logger::testLog << L"Number of unique errors: " << werr_stream_buf.get_unique_errors() << L"\n";
|
||||
|
||||
if (user_opt.stress) {
|
||||
Logger::testLog.disable();
|
||||
Logger::testLog.flush();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
catch(const Ort::Exception& ortException)
|
||||
{
|
||||
Logger::testLog<< L"onnx runtime exception: " << ortException.what() << Logger::endl;
|
||||
}
|
||||
catch(const std::exception& e)
|
||||
{
|
||||
Logger::testLog<< L"standard exception: " << e.what() << Logger::endl;
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
Logger::testLog<< L"Something Went very wrong: " << Logger::endl;
|
||||
} catch (const Ort::Exception& ort_exception) {
|
||||
Logger::testLog << L"onnx runtime exception: " << ort_exception.what() << Logger::endl;
|
||||
} catch (const std::exception& e) {
|
||||
Logger::testLog << L"standard exception: " << e.what() << Logger::endl;
|
||||
} catch (...) {
|
||||
Logger::testLog << L"Something Went very wrong: " << Logger::endl;
|
||||
}
|
||||
|
||||
return 1;
|
||||
|
|
|
|||
157
tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
Normal file
157
tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
jobs:
|
||||
- job: 'build'
|
||||
pool: 'Win-CPU-2019'
|
||||
strategy:
|
||||
maxParallel: 2
|
||||
matrix:
|
||||
debug:
|
||||
BuildConfig: 'Debug'
|
||||
release:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
variables:
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
|
||||
MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary'
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
setVcvars: true
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
|
||||
timeoutInMinutes: 120
|
||||
workspace:
|
||||
clean: all
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.7'
|
||||
addToPath: true
|
||||
architecture: $(buildArch)
|
||||
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: '12.x'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup env'
|
||||
inputs:
|
||||
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
||||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- script: |
|
||||
python -m pip install -q pyopenssl setuptools wheel numpy flake8
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
displayName: 'Install python modules'
|
||||
|
||||
- powershell: |
|
||||
$Env:USE_MSVC_STATIC_RUNTIME=1
|
||||
$Env:ONNX_ML=1
|
||||
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
||||
python setup.py bdist_wheel
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
||||
displayName: 'Install ONNX'
|
||||
|
||||
- task: NuGetToolInstaller@0
|
||||
displayName: Use Nuget 5.7.0
|
||||
inputs:
|
||||
versionSpec: 5.7.0
|
||||
|
||||
- task: NuGetCommand@2
|
||||
displayName: 'NuGet restore'
|
||||
inputs:
|
||||
command: 'restore'
|
||||
feedsToUse: 'config'
|
||||
restoreSolution: '$(Build.SourcesDirectory)\packages.config'
|
||||
nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
|
||||
restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--gen_doc --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java --build_nodejs --use_full_protobuf --fuzz_testing'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
|
||||
platform: 'x64'
|
||||
configuration: $(BuildConfig)
|
||||
msbuildArgs: $(MsbuildArguments)
|
||||
msbuildArchitecture: $(buildArch)
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
createLogFile: true
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
|
||||
- template: templates/set-test-data-variables-step.yml
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: 'Restore nuget packages'
|
||||
inputs:
|
||||
command: restore
|
||||
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
configuration: '$(BuildConfig)'
|
||||
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: 'Build C#'
|
||||
inputs:
|
||||
command: build
|
||||
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
configuration: '$(BuildConfig)'
|
||||
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: 'Test C#'
|
||||
condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))
|
||||
inputs:
|
||||
command: test
|
||||
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
|
||||
configuration: '$(BuildConfig)'
|
||||
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- script: |
|
||||
mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models
|
||||
DIR dist\ /S /B > wheel_filename_file
|
||||
set /p WHEEL_FILENAME=<wheel_filename_file
|
||||
del wheel_filename_file
|
||||
python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
|
||||
set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH%
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_nodejs --test --cmake_generator "Visual Studio 16 2019" --use_dnnl --build_wheel --enable_onnx_tests --use_full_protobuf --fuzz_testing
|
||||
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
displayName: 'Run tests'
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
- task: CmdLine@2
|
||||
displayName: 'Run fuzz testing '
|
||||
inputs:
|
||||
script: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\onnxruntime_security_fuzz.exe /t /f "$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\testdata\mnist.onnx" 1 m'
|
||||
workingDirectory: $(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)
|
||||
failOnStderr: false # Optional
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
Loading…
Reference in a new issue