From b495ae81031e6d98199d15652ed1ff6bdcd56b8b Mon Sep 17 00:00:00 2001 From: satyajandhyala Date: Wed, 18 Nov 2020 16:07:36 -0800 Subject: [PATCH] ORT fuzz testing (#5771) * Added fuzz testing using ORT model. * The onnxruntime_security_fuzz driver code should accept either ONNX or ORT (based on the file extension) input file if /f flag is provided. * Added ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild test. * Added win-ci-fuzz-testing.yml to run build pipeline. * Prevent out-of-range access in the graph.cpp. --- onnxruntime/core/graph/graph.cc | 2 + .../test/framework/ort_model_only_test.cc | 35 ++ .../test/fuzzing/include/OnnxPrediction.h | 137 ++--- .../test/fuzzing/src/OnnxPrediction.cpp | 213 +++---- onnxruntime/test/fuzzing/src/test.cpp | 567 +++++++++--------- .../azure-pipelines/win-ci-fuzz-testing.yml | 157 +++++ 6 files changed, 622 insertions(+), 489 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index f2217e4b85..3f14d89a96 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -3732,6 +3732,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr ORT_RETURN_IF(nullptr == fbs_node, "Node is missing. Invalid ORT format model."); std::unique_ptr node; ORT_RETURN_IF_ERROR(Node::LoadFromOrtFormat(*fbs_node, *this, logger_, node)); + ORT_RETURN_IF(node->Index() >= fbs_graph.max_node_index(), "Node index is out of range"); nodes_[node->Index()] = std::move(node); ++num_of_nodes_; } @@ -3742,6 +3743,7 @@ common::Status Graph::LoadFromOrtFormat(const onnxruntime::experimental::fbs::Gr if (fbs_node_edges != nullptr) { for (const auto* fbs_node_edge : *fbs_node_edges) { ORT_RETURN_IF(nullptr == fbs_node_edge, "NodeEdge is missing. Invalid ORT format model."); + ORT_RETURN_IF(fbs_node_edge->node_index() >= fbs_graph.max_node_index(), "Node index is out of range"); ORT_RETURN_IF_ERROR(nodes_[fbs_node_edge->node_index()]->LoadEdgesFromOrtFormat(*fbs_node_edge, *this)); } } diff --git a/onnxruntime/test/framework/ort_model_only_test.cc b/onnxruntime/test/framework/ort_model_only_test.cc index 3510cbb5f2..3253981b51 100644 --- a/onnxruntime/test/framework/ort_model_only_test.cc +++ b/onnxruntime/test/framework/ort_model_only_test.cc @@ -237,6 +237,41 @@ static void DumpOrtModelAsJson(const std::string& model_uri) { std::ofstream(model_uri + ".json") << json; } */ +/* The full build was causing the following error because the graph node array has some empyt (blank) node at some indices for certain ORT designs + onnx runtime exception : Satisfied, but should not be : node == nullptr + session_state.cc : 814 onnxruntime::SessionState::LoadFromOrtFormatCan't find node with index 4. Invalid ORT format model. + The bug was due to loading an ORT format model in a full build, allowing optimizers to run, but trying to use the saved kernel information. + As the optimizer removed some leaving gaps in the graph node vector. + The build has been fixed in InferenceSession code. The following test case to catch this error. +*/ + +TEST(OrtModelOnlyTests, ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild) { + const std::basic_string ort_file = ORT_TSTR("mnist.onnx.ort"); + SaveAndCompareModels("testdata/mnist.onnx", ort_file); + + // DumpOrtModelAsJson(ToMBString(ort_file)); + + OrtModelTestInfo test_info; + test_info.model_filename = ort_file; + test_info.logid = "ValidateOrtFormatModelDoesNotRunOptimizersInFullBuild"; + test_info.configs.push_back(std::make_pair(kOrtSessionOptionsConfigLoadModelFormat, "ORT")); + + OrtValue ml_value; + vector data(28*28, 0.0); + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), {1,1,28,28}, data, + &ml_value); + test_info.inputs.insert(std::make_pair("Input3", ml_value)); + + // prepare outputs + test_info.output_names = {"Plus214_Output_0"}; + test_info.output_verifier = [](const std::vector& fetches) { + const auto& output = fetches[0].Get(); + ASSERT_TRUE(output.Shape().NumDimensions() == 2); + // ASSERT_TRUE(output.Data()[0] == 125.f); + }; + + RunOrtModel(test_info); +} TEST(OrtModelOnlyTests, SerializeToOrtFormat) { const std::basic_string ort_file = ORT_TSTR("ort_github_issue_4031.onnx.ort"); diff --git a/onnxruntime/test/fuzzing/include/OnnxPrediction.h b/onnxruntime/test/fuzzing/include/OnnxPrediction.h index b5938e28a9..687614af91 100644 --- a/onnxruntime/test/fuzzing/include/OnnxPrediction.h +++ b/onnxruntime/test/fuzzing/include/OnnxPrediction.h @@ -25,82 +25,74 @@ #include "testlog.h" -class OnnxPrediction -{ +class OnnxPrediction { friend std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred); friend Logger::TestLog& Logger::TestLog::operator<<(OnnxPrediction& pred); -public: - using InputGeneratorFunctionType = std::function; -public: + public: + using InputGeneratorFunctionType = std::function; + + public: // Uses the onnxruntime to load the model // into a session. // - OnnxPrediction(std::wstring& onnx_model_file); + OnnxPrediction(std::wstring& onnx_model_file, Ort::Env& env); // Uses the onnx model to create a prediction // environment // - OnnxPrediction(onnx::ModelProto& onnx_model); + OnnxPrediction(onnx::ModelProto& onnx_model, Ort::Env& env); - // Deletes the prediction object + // The following constructor is meant for initializing using flatbuffer model. + // Memory buffer pointing to the model // - ~OnnxPrediction(); + OnnxPrediction(const std::vector& model_data, Ort::Env& env); // Data to run prediction on // - template - void operator<<(std::vector&& raw_data) - { - if( currInputIndex >= ptrSession->GetInputCount()) - { + template + void operator<<(std::vector&& raw_data) { + if (curr_input_index >= ptr_session->GetInputCount()) { return; } - Ort::Value& inputValue = inputValues[currInputIndex]; + Ort::Value& input_value = input_values[curr_input_index]; auto data_size_in_bytes = raw_data.size() * sizeof(T); - + // TODO The following allocation may be unnecessary // Copy the raw input data and control the lifetime. // - inputData.emplace_back(alloc.Alloc(data_size_in_bytes), - [this](void * ptr1) - { - this->GetAllocator().Free(ptr1); - } - ); - - std::copy(raw_data.begin(), raw_data.end(), reinterpret_cast(inputData[currInputIndex].get())); - auto inputType = ptrSession->GetInputTypeInfo(currInputIndex); - auto shapeInfo = inputType.GetTensorTypeAndShapeInfo().GetShape(); - auto elem_type = inputType.GetTensorTypeAndShapeInfo().GetElementType(); - if ( elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) - { - inputValue = Ort::Value::CreateTensor(alloc.GetInfo(), - inputData[currInputIndex].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type); - } - else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) - { - inputValue = Ort::Value::CreateTensor(alloc.GetInfo(), - inputData[currInputIndex].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type); - } - else - { + input_data.emplace_back(alloc.Alloc(data_size_in_bytes), + [this](void* ptr1) { + this->GetAllocator().Free(ptr1); + }); + + std::copy(raw_data.begin(), raw_data.end(), reinterpret_cast(input_data[curr_input_index].get())); + auto input_type = ptr_session->GetInputTypeInfo(curr_input_index); + auto shapeInfo = input_type.GetTensorTypeAndShapeInfo().GetShape(); + auto elem_type = input_type.GetTensorTypeAndShapeInfo().GetElementType(); + if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + input_value = Ort::Value::CreateTensor(alloc.GetInfo(), + input_data[curr_input_index].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type); + } else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) { + input_value = Ort::Value::CreateTensor(alloc.GetInfo(), + input_data[curr_input_index].get(), data_size_in_bytes, shapeInfo.data(), shapeInfo.size(), elem_type); + } else { throw std::exception("only floats are implemented"); } // Insert data into the next input type // - currInputIndex++; + curr_input_index++; } - // Used to generate + // Used to generate // void SetupInput( - InputGeneratorFunctionType GenerateData, - size_t seed = static_cast(std::chrono::system_clock::now().time_since_epoch().count())); + InputGeneratorFunctionType GenerateData, + size_t seed = static_cast(std::chrono::system_clock::now().time_since_epoch().count())); // Run the prediction // @@ -108,20 +100,13 @@ public: // Do operation on the output data // - template - void ProcessOutputData(T process_function, Ort::Value& val) - { - if ( val.IsTensor() ) - { - if (val.GetTensorTypeAndShapeInfo().GetElementType() - == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) - { + template + void ProcessOutputData(T process_function, Ort::Value& val) { + if (val.IsTensor()) { + if (val.GetTensorTypeAndShapeInfo().GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { auto ptr = val.GetTensorMutableData(); process_function(ptr, val); - } - else if (val.GetTensorTypeAndShapeInfo().GetElementType() - == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) - { + } else if (val.GetTensorTypeAndShapeInfo().GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) { auto ptr = val.GetTensorMutableData(); process_function(ptr, val); } @@ -132,7 +117,7 @@ public: // void PrintOutputValues(); -private: + private: // Common initilization amongst constructors // void init(); @@ -141,59 +126,51 @@ private: // Ort::AllocatorWithDefaultOptions& GetAllocator(); -private: + private: // Create an allocator for the runtime to use // Ort::AllocatorWithDefaultOptions alloc; - // Create Environment - // - Ort::Env env; - // Create Options for the Session // - Ort::SessionOptions emptySessionOption; + Ort::SessionOptions empty_session_option; // A pointer to the model // - std::shared_ptr rawModel; + std::shared_ptr raw_model; // Create RunOptions // - Ort::RunOptions runOptions; - - // Create a Session to run - // - Ort::Session session; + Ort::RunOptions run_options; // Pointer to the current session object // - std::unique_ptr ptrSession; + std::unique_ptr ptr_session; // Create a list of input values // - std::vector inputValues{}; + std::vector input_values{}; // Stores the input names // - std::vector inputNames; + std::vector input_names; // Stores the output names // - std::vector outputNames; + std::vector output_names; // Create a list of output values // - std::vector outputValues{}; + std::vector output_values{}; // We own the lifetime of the input data // - std::vector> inputData; + std::vector> input_data; // Keeps track of number of columns/dimensions data // given for predicition // - size_t currInputIndex{0}; + size_t curr_input_index{0}; }; // OnnxPrediction console output format @@ -204,6 +181,6 @@ std::ostream& operator<<(std::ostream& out, OnnxPrediction& pred); // Used to Generate data for predict // void GenerateDataForInputTypeTensor(OnnxPrediction& predict, - size_t input_index, const std::string& input_name, - ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed); + size_t input_index, const std::string& input_name, + ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed); #endif diff --git a/onnxruntime/test/fuzzing/src/OnnxPrediction.cpp b/onnxruntime/test/fuzzing/src/OnnxPrediction.cpp index 5edc385ed1..fa52089f71 100644 --- a/onnxruntime/test/fuzzing/src/OnnxPrediction.cpp +++ b/onnxruntime/test/fuzzing/src/OnnxPrediction.cpp @@ -2,144 +2,125 @@ // Licensed under the MIT License. #include "OnnxPrediction.h" +#include "onnxruntime_session_options_config_keys.h" + +#include // Uses the onnxruntime to load the model // into a session. // -OnnxPrediction::OnnxPrediction(std::wstring& onnx_model_file) -: -rawModel{nullptr}, -ptrSession{nullptr}, -session{ env, onnx_model_file.c_str(), emptySessionOption}, -inputNames{session.GetInputCount()}, -outputNames{session.GetOutputCount()} -{ +OnnxPrediction::OnnxPrediction(std::wstring& onnx_model_file, Ort::Env& env) + : raw_model{nullptr}, + ptr_session{std::make_unique(env, onnx_model_file.c_str(), empty_session_option)}, + input_names(ptr_session->GetInputCount()), + output_names(ptr_session->GetOutputCount()) { init(); } // Uses the onnx to seri // -OnnxPrediction::OnnxPrediction(onnx::ModelProto& onnx_model) -: -session{nullptr} -{ - rawModel = std::shared_ptr{alloc.Alloc(onnx_model.ByteSizeLong()), - [this](void * ptr) - { - this->GetAllocator().Free(ptr); - } - }; +OnnxPrediction::OnnxPrediction(onnx::ModelProto& onnx_model, Ort::Env& env) { + raw_model = std::shared_ptr(alloc.Alloc(onnx_model.ByteSizeLong()), [this](void* ptr) { + this->GetAllocator().Free(ptr); + }); - onnx_model.SerializeToArray(rawModel.get(), static_cast(onnx_model.ByteSizeLong())); + onnx_model.SerializeToArray(raw_model.get(), static_cast(onnx_model.ByteSizeLong())); - ptrSession = std::make_unique(env, - rawModel.get(), - onnx_model.ByteSizeLong(), - emptySessionOption), + ptr_session = std::make_unique(env, + static_cast(raw_model.get()), + onnx_model.ByteSizeLong(), + empty_session_option); - inputNames.resize(ptrSession->GetInputCount()); - outputNames.resize(ptrSession->GetOutputCount()); + // TODO Use reserve instead of resize + input_names.resize(ptr_session->GetInputCount()); + output_names.resize(ptr_session->GetOutputCount()); init(); } -// Destructor -// -OnnxPrediction::~OnnxPrediction() -{ - if (ptrSession.get() == &session) - { - // Ensure the session is not deleted - // by the unique_ptr. Because it will be deleting the stack - // - ptrSession.release(); - } +OnnxPrediction::OnnxPrediction(const std::vector& model_data, Ort::Env& env) { + Ort::SessionOptions so; + so.AddConfigEntry(kOrtSessionOptionsConfigLoadModelFormat, "ORT"); + ptr_session = std::make_unique(env, + model_data.data(), + model_data.size(), + so); + + // TODO Use reserve instead of resize + input_names.resize(ptr_session->GetInputCount()); + output_names.resize(ptr_session->GetOutputCount()); + + init(); } // OnnxPrediction console output format // prints the output data. // -std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred) -{ - auto pretty_print = [&out](auto ptr, Ort::Value& val) - { - out << L"["; - std::wstring msg = L""; - for(int i=0; i < val.GetTensorTypeAndShapeInfo().GetElementCount(); i++) - { - out << msg << ptr[i]; - msg = L", "; - } - out << L"]\n"; +std::wostream& operator<<(std::wostream& out, OnnxPrediction& pred) { + auto pretty_print = [&out](auto ptr, Ort::Value& val) { + out << L"["; + std::wstring msg = L""; + for (int i = 0; i < val.GetTensorTypeAndShapeInfo().GetElementCount(); i++) { + out << msg << ptr[i]; + msg = L", "; + } + out << L"]\n"; }; - size_t index {0}; - for(auto& val : pred.outputValues) - { - out << pred.outputNames[index++] << L" = "; + size_t index{0}; + for (auto& val : pred.output_values) { + out << pred.output_names[index++] << L" = "; pred.ProcessOutputData(pretty_print, val); } - + return out; } // Used to Generate data for predict // void GenerateDataForInputTypeTensor(OnnxPrediction& predict, - size_t input_index, const std::string& input_name, - ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed) -{ - (void) input_name; - (void) input_index; - - auto pretty_print = [&input_name](auto raw_data) - { + size_t input_index, const std::string& input_name, + ONNXTensorElementDataType elem_type, size_t elem_count, size_t seed) { + (void)input_name; + (void)input_index; + + auto pretty_print = [&input_name](auto raw_data) { Logger::testLog << input_name << L" = "; Logger::testLog << L"["; std::wstring msg = L""; - for(int i=0; i < raw_data.size(); i++) - { + for (int i = 0; i < raw_data.size(); i++) { Logger::testLog << msg << raw_data[i]; msg = L", "; } - Logger::testLog << L"]\n"; + Logger::testLog << L"]\n"; }; - if ( elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) - { - auto raw_data = GenerateRandomData(0.0f, elem_count,seed); - pretty_print(raw_data); + if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + auto raw_data = GenerateRandomData(0.0f, elem_count, seed); + // pretty_print(raw_data); predict << std::move(raw_data); - } - else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) - { + } else if (elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) { int32_t initial = 0; - auto raw_data = GenerateRandomData(initial, elem_count,seed); - pretty_print(raw_data); + auto raw_data = GenerateRandomData(initial, elem_count, seed); + // pretty_print(raw_data); predict << std::move(raw_data); - } - else - { + } else { throw std::exception("only floats are implemented"); } } // Run the prediction // -void OnnxPrediction::RunInference() -{ +void OnnxPrediction::RunInference() { Logger::testLog << L"inference starting " << Logger::endl; Logger::testLog.flush(); - try - { - ptrSession->Run(runOptions, - inputNames.data(), inputValues.data(), - inputValues.size(), outputNames.data(), outputValues.data(), - outputNames.size()); - } - catch(...) - { + try { + ptr_session->Run(run_options, + input_names.data(), input_values.data(), + input_values.size(), output_names.data(), output_values.data(), + output_names.size()); + } catch (...) { Logger::testLog << L"Something went wrong in inference " << Logger::endl; Logger::testLog.flush(); throw; @@ -151,8 +132,7 @@ void OnnxPrediction::RunInference() // Print the output values of the prediction. // -void OnnxPrediction::PrintOutputValues() -{ +void OnnxPrediction::PrintOutputValues() { Logger::testLog << L"output data:\n"; Logger::testLog << *this; Logger::testLog << Logger::endl; @@ -160,68 +140,49 @@ void OnnxPrediction::PrintOutputValues() // Common initilization amongst constructors // -void OnnxPrediction::init() -{ - // Enable telemetry events - // - env.EnableTelemetryEvents(); - - if (!ptrSession) - { - // To use one consistent value - // across the class - // - ptrSession.reset(&session); - } - +void OnnxPrediction::init() { // Initialize model input names // - for(int i=0; i < ptrSession->GetInputCount(); i++) - { - inputNames[i] = ptrSession->GetInputName(i, alloc); - inputValues.emplace_back(nullptr); + for (int i = 0; i < ptr_session->GetInputCount(); i++) { + // TODO Use push_back on input_names instead of assignment + input_names[i] = ptr_session->GetInputName(i, alloc); + input_values.emplace_back(nullptr); } // Initialize model output names // - for(int i=0; i < ptrSession->GetOutputCount(); i++) - { - outputNames[i] = ptrSession->GetOutputName(i, alloc); - outputValues.emplace_back(nullptr); + for (int i = 0; i < ptr_session->GetOutputCount(); i++) { + // TODO Use push_back on output_names instead of assignment + output_names[i] = ptr_session->GetOutputName(i, alloc); + output_values.emplace_back(nullptr); } } // Get the allocator used by the runtime // -Ort::AllocatorWithDefaultOptions& OnnxPrediction::GetAllocator() -{ +Ort::AllocatorWithDefaultOptions& OnnxPrediction::GetAllocator() { return alloc; } void OnnxPrediction::SetupInput( - InputGeneratorFunctionType GenerateData, - size_t seed) -{ + InputGeneratorFunctionType GenerateData, + size_t seed) { Logger::testLog << L"input data:\n"; - for(int i=0; i < ptrSession->GetInputCount(); i++) - { - auto inputType = ptrSession->GetInputTypeInfo(i); + for (int i = 0; i < ptr_session->GetInputCount(); i++) { + auto inputType = ptr_session->GetInputTypeInfo(i); - if (inputType.GetONNXType() == ONNX_TYPE_TENSOR) - { + if (inputType.GetONNXType() == ONNX_TYPE_TENSOR) { auto elem_type = inputType.GetTensorTypeAndShapeInfo().GetElementType(); auto elem_count = inputType.GetTensorTypeAndShapeInfo().GetElementCount(); - + // This can be any generic function to generate inputs // - GenerateData(*this, i , std::string(inputNames[i]), elem_type, elem_count, seed); + GenerateData(*this, i, std::string(input_names[i]), elem_type, elem_count, seed); // Update the seed in a predicatable way to get other values for different inputs // seed++; - } - else - { + } else { std::cout << "Unsupported \n"; } } diff --git a/onnxruntime/test/fuzzing/src/test.cpp b/onnxruntime/test/fuzzing/src/test.cpp index 3e747597cf..b15e174264 100644 --- a/onnxruntime/test/fuzzing/src/test.cpp +++ b/onnxruntime/test/fuzzing/src/test.cpp @@ -4,191 +4,176 @@ #include "src/mutator.h" #include "testlog.h" #include "OnnxPrediction.h" +#include "onnxruntime_session_options_config_keys.h" + #include -using userOptions = struct +using user_options = struct { - bool writeModel; + bool write_model; bool verbose; bool stress; + bool is_ort; }; -void predict(onnx::ModelProto& model_proto, unsigned int seed) -{ - // Create object for prediction - // - OnnxPrediction predict{model_proto}; +void predict(onnx::ModelProto& model_proto, unsigned int seed, Ort::Env& env) { + // Create object for prediction + // + OnnxPrediction predict(model_proto, env); - // Give predict a function to generate the data - // to run prediction on. - // - predict.SetupInput(GenerateDataForInputTypeTensor, seed); + // Give predict a function to generate the data + // to run prediction on. + // + predict.SetupInput(GenerateDataForInputTypeTensor, seed); - // Run the prediction on the data - // - predict.RunInference(); + // Run the prediction on the data + // + predict.RunInference(); - // View the output - // - predict.PrintOutputValues(); + // View the output + // + predict.PrintOutputValues(); } -void mutateModelTest(onnx::ModelProto& model_proto, - std::wstring mutatedModelDirName, - userOptions opt, - unsigned int seed = 0) -{ - // Used to initialize all random engines +void mutateModelTest(onnx::ModelProto& model_proto, + std::wstring mutatedModelDirName, + user_options opt, + Ort::Env& env, + unsigned int seed = 0) { + // Used to initialize all random engines + // + std::wstring modelPrefix = L"/ReproMutateModel_"; + if (seed == 0) { + seed = static_cast(std::chrono::system_clock::now().time_since_epoch().count()); + modelPrefix = L"/MutateModel_"; + } + + if (opt.stress) { + Logger::testLog.enable(); + } + + Logger::testLog << L"Mutate test seed: " << seed << Logger::endl; + opt.stress ? Logger::testLog.disable() : Logger::testLog.enable(); + + // Create mutator + // + protobuf_mutator::Mutator mutator; + + // Mutate model + // + Logger::testLog << L"Model Successfully Initialized" << Logger::endl; + mutator.Seed(seed); + mutator.Mutate(&model_proto, model_proto.ByteSizeLong()); + + if (opt.write_model) { + // Create file to store model // - std::wstring modelPrefix = L"/ReproMutateModel_"; - if(seed == 0) - { - seed = static_cast(std::chrono::system_clock::now().time_since_epoch().count()); - modelPrefix = L"/MutateModel_"; - } - - if(opt.stress) - { - Logger::testLog.enable(); - } + std::wstringstream mutateModelName; + mutateModelName << mutatedModelDirName << modelPrefix << seed << L".onnx"; + auto mutateModelFileName = mutateModelName.str(); - Logger::testLog << L"Mutate test seed: " << seed << Logger::endl; - opt.stress ? Logger::testLog.disable() : Logger::testLog.enable(); - - // Create mutator + // Log the model to a file // - protobuf_mutator::Mutator mutator; - - // Mutate model + std::ofstream outStream(mutateModelFileName); + model_proto.SerializeToOstream(&outStream); + Logger::testLog << "Mutated Model Written to file: " << mutateModelFileName << Logger::endl; + + // Flush the buffer to ensure the + // mutated model info for reproduction + // purposes. // - Logger::testLog << L"Model Successfully Initialized" << Logger::endl; - mutator.Seed(seed); - mutator.Mutate(&model_proto, model_proto.ByteSizeLong()); + outStream << std::flush; + } - if (opt.writeModel) - { - // Create file to store model - // - std::wstringstream mutateModelName; - mutateModelName << mutatedModelDirName << modelPrefix << seed << L".onnx"; - auto mutateModelFileName = mutateModelName.str(); + // Flush any logs before prediction + // + Logger::testLog.flush(); - // Log the model to a file - // - std::ofstream outStream(mutateModelFileName); - model_proto.SerializeToOstream(&outStream); - Logger::testLog<< "Mutated Model Written to file: " << mutateModelFileName << Logger::endl; + // run prediction on model + // + predict(model_proto, seed, env); - // Flush the buffer to ensure the - // mutated model info for reproduction - // purposes. - // - outStream << std::flush; - } - - // Flush any logs before prediction - // - Logger::testLog.flush(); - - // run prediction on model - // - predict(model_proto, seed); - - // print out all output before next test - // - Logger::testLog.flush(); + // print out all output before next test + // + Logger::testLog.flush(); } -void printUsage() -{ +void printUsage() { std::cout << "Not enough command line arguments\n"; std::cout << "usage:\n" - << "\t\tFor testing: test.exe /t [options] onnx_model_file test_timeout test_time_scale\n" - << "\t\tFor repro/debugging: test.exe /r onnx_model_file seed\n" - << "\n\nonnx_model_file: Unmutated onnx model file\n" - << "options: /m - output mutated models /v - verbose logging /s - stress test" - << "test_time_scale: h|m|s\n" - << "test_timeout: Time to run the test in hrs\n" - << "seed: The seed that generated the mutated model. This value is the decimal digit part of the mutated model name (or can be found in the logs)\n" - << "\n"; + << "\t\tFor testing: test.exe /t [options] onnx_model_file test_timeout test_time_scale\n" + << "\t\tFor repro/debugging: test.exe /r onnx_model_file seed\n" + << "\n\nonnx_model_file: Unmutated onnx model file\n" + << "options: /m - output mutated models /v - verbose logging /s - stress test" + << "test_time_scale: h|m|s\n" + << "test_timeout: Time to run the test in hrs\n" + << "seed: The seed that generated the mutated model. This value is the decimal digit part of the mutated model name (or can be found in the logs)\n" + << "\n"; } -enum class timeScale : char -{ - Hrs = 'h', +enum class timeScale : char { + Hrs = 'h', Min = 'm', Sec = 's' }; -using runtimeOpt = struct -{ - std::wstring modelFileName{}; - std::wstring mutateModelDirName{}; - Logger::ccstream errStreamBuf{}; - Logger::wcstream werrStreamBuf{}; - bool repoMode{false}; - int testTimeOut{0}; +struct runtimeOpt { + std::wstring model_file_name{}; + std::wstring mutate_model_dir_name{}; + Logger::ccstream err_stream_buf{}; + Logger::wcstream werr_stream_buf{}; + bool repo_mode{false}; + int test_time_out{0}; unsigned int seed{0}; timeScale scale{timeScale::Sec}; - userOptions userOpt{false, false, false}; + user_options user_opt{false, false, false, false}; }; -int processCommandLine(int argc, char* argv[], runtimeOpt& opt) -{ - if (argc <= 1) - { +int processCommandLine(int argc, char* argv[], runtimeOpt& opt) { + if (argc <= 1) { printUsage(); return 2; - } - else - { + } else { bool isTest = std::string{argv[1]} == "/t"; bool isRepo = std::string{argv[1]} == "/r"; - if(isRepo) - { - opt.repoMode = true; - opt.mutateModelDirName = L"./repromodel"; - std::filesystem::path mutateModelDir{opt.mutateModelDirName}; - if ( !std::filesystem::exists(mutateModelDir) ) - { - std::filesystem::create_directory(mutateModelDir); + if (isRepo) { + opt.repo_mode = true; + opt.mutate_model_dir_name = L"./repromodel"; + std::filesystem::path mutate_model_dir{opt.mutate_model_dir_name}; + if (!std::filesystem::exists(mutate_model_dir)) { + std::filesystem::create_directory(mutate_model_dir); } - opt.modelFileName = Logger::towstr(argv[2]); - Logger::testLog<< L"Repo Model file: " << opt.modelFileName << Logger::endl; + opt.model_file_name = Logger::towstr(argv[2]); + Logger::testLog << L"Repo Model file: " << opt.model_file_name << Logger::endl; // Get seed // std::stringstream parser{argv[3]}; parser >> opt.seed; - if (parser.bad()) - { + if (parser.bad()) { throw std::exception("Could not parse seed from command line"); } - std::wcout << L"seed: " << opt.seed << L"\n"; - } - else if (isTest) - { + std::wcout << L"seed: " << opt.seed << L"\n"; + } else if (isTest) { int index{argc}; index--; // Parse right to left // std::stringstream parser; - char desiredScale; + char desired_scale; parser << argv[index--]; - parser >> desiredScale; - - if (parser.bad()) - { + parser >> desired_scale; + + if (parser.bad()) { throw std::exception("Could not parse the time scale from the command line"); } - - opt.scale = static_cast(std::tolower(desiredScale)); - switch(opt.scale) - { + + opt.scale = static_cast(std::tolower(desired_scale)); + switch (opt.scale) { case timeScale::Hrs: case timeScale::Min: case timeScale::Sec: @@ -196,67 +181,56 @@ int processCommandLine(int argc, char* argv[], runtimeOpt& opt) default: throw std::exception("Could not parse the time scale from the command line"); } - + parser << argv[index--]; - parser >> opt.testTimeOut; - if (parser.bad()) - { + parser >> opt.test_time_out; + if (parser.bad()) { throw std::exception("Could not parse the time value from the command line"); } - Logger::testLog<< L"Running Test for: " << opt.testTimeOut << desiredScale << Logger::endl; - opt.modelFileName = Logger::towstr(argv[index--]); - Logger::testLog<< L"Model file: " << opt.modelFileName << Logger::endl; - std::filesystem::path modelFileNamePath{opt.modelFileName}; - if (!std::filesystem::exists(modelFileNamePath)) - { + Logger::testLog << L"Running Test for: " << opt.test_time_out << desired_scale << Logger::endl; + opt.model_file_name = Logger::towstr(argv[index--]); + Logger::testLog << L"Model file: " << opt.model_file_name << Logger::endl; + std::filesystem::path model_file_namePath{opt.model_file_name}; + if (!std::filesystem::exists(model_file_namePath)) { throw std::exception("Cannot find model file"); } // process options // - while(index > 0) - { + while (index > 0) { auto option{std::string{argv[index]}}; - if ( option == "/m") - { - opt.userOpt.writeModel = true; - } - else if (option == "/v") - { - opt.userOpt.verbose = true; - } - else if (option == "/s") - { - opt.userOpt.stress = true; + if (option == "/m") { + opt.user_opt.write_model = true; + } else if (option == "/v") { + opt.user_opt.verbose = true; + } else if (option == "/s") { + opt.user_opt.stress = true; + } else if (option == "/f") { + opt.user_opt.is_ort = true; } index--; } - if (opt.userOpt.stress) - { - std::cerr.rdbuf(&opt.errStreamBuf); - std::wcerr.rdbuf(&opt.werrStreamBuf); - opt.userOpt.writeModel = false; - opt.userOpt.verbose = false; + if (opt.user_opt.stress) { + std::cerr.rdbuf(&opt.err_stream_buf); + std::wcerr.rdbuf(&opt.werr_stream_buf); + opt.user_opt.write_model = false; + opt.user_opt.verbose = false; Logger::testLog.disable(); Logger::testLog.minLog(); } - // create directory for mutated model output + // create directory for mutated model output // - if(opt.userOpt.writeModel) - { - opt.mutateModelDirName = L"./mutatemodel"; - std::filesystem::path mutateModelDir{opt.mutateModelDirName}; - if ( !std::filesystem::exists(mutateModelDir) ) - { - std::filesystem::create_directory(mutateModelDir); + if (opt.user_opt.write_model) { + opt.mutate_model_dir_name = L"./mutatemodel"; + std::filesystem::path mutate_model_dir{opt.mutate_model_dir_name}; + if (!std::filesystem::exists(mutate_model_dir)) { + std::filesystem::create_directory(mutate_model_dir); } } - } - else - { + } else { printUsage(); return 2; } @@ -265,143 +239,170 @@ int processCommandLine(int argc, char* argv[], runtimeOpt& opt) return 0; } -int main(int argc, char* argv[]) -{ +struct RunStats { + size_t num_ort_exception; + size_t num_std_exception; + size_t num_unknown_exception; + size_t num_successful_runs; + size_t iteration; +}; + +static void fuzz_handle_exception(struct RunStats& run_stats) { + try { + throw; + } catch (const Ort::Exception& ortException) { + run_stats.num_ort_exception++; + Logger::testLog << L"onnx runtime exception: " << ortException.what() << Logger::endl; + Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl; + } catch (const std::exception& e) { + run_stats.num_std_exception++; + Logger::testLog << L"standard exception: " << e.what() << Logger::endl; + Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl; + } catch (...) { + run_stats.num_unknown_exception++; + Logger::testLog << L"unknown exception: " << Logger::endl; + Logger::testLog << "Failed Test iteration: " << run_stats.iteration++ << Logger::endl; + throw; + } +} + +int main(int argc, char* argv[]) { + Ort::Env env; + // Enable telemetry events + // + env.EnableTelemetryEvents(); + struct RunStats run_stats = {0, 0, 0, 0, 0}; runtimeOpt opt{}; - try - { + user_options& user_opt{opt.user_opt}; + Logger::wcstream& werr_stream_buf{opt.werr_stream_buf}; + try { // Initialize the runtime options // auto canContinue{processCommandLine(argc, argv, opt) == 0}; - if(!canContinue) - { + if (!canContinue) { return -1; } - std::wstring& modelFileName{opt.modelFileName}; - std::wstring& mutateModelDirName{opt.mutateModelDirName}; - bool& repoMode{opt.repoMode}; - int& testTimeOut{opt.testTimeOut}; + std::wstring& model_file_name{opt.model_file_name}; + std::wstring& mutate_model_dir_name{opt.mutate_model_dir_name}; + bool& repo_mode{opt.repo_mode}; + int& test_time_out{opt.test_time_out}; unsigned int& seed{opt.seed}; timeScale& scale{opt.scale}; - userOptions& userOpt{opt.userOpt}; - Logger::wcstream& werrStreamBuf{opt.werrStreamBuf}; - + // Model file // - std::wstring modelFile {modelFileName}; - + std::wstring model_file{model_file_name}; + // Create a stream to hold the model // - std::ifstream modelStream{modelFile}; + std::ifstream modelStream{model_file, std::ios::in | std::ios::binary}; + if (opt.user_opt.is_ort == false) { + // Create an onnx protobuf object + // + onnx::ModelProto model_proto; - // Create an onnx protobuf object - // - onnx::ModelProto model_proto; - - // Initialize the model - // - if( model_proto.ParseFromIstream(&modelStream) ) - { - if(repoMode) - { - Logger::testLog<< L"Running Prediction for: " << modelFileName - << L" with seed " << seed << Logger::endl; - mutateModelTest(model_proto, mutateModelDirName, userOpt, seed); - Logger::testLog<< L"Finished Prediction for: " << modelFileName - << L" with seed " << seed << Logger::endl; - } - else - { - // Call the mutateModelTest - // - std::chrono::system_clock::time_point currTime{ std::chrono::system_clock::now() }; - - std::chrono::minutes timeInMin{testTimeOut}; - std::chrono::seconds timeInSec{testTimeOut}; - std::chrono::hours timeInHrs{testTimeOut}; - std::chrono::system_clock::time_point endTime{currTime}; - endTime += scale == timeScale::Hrs ? timeInHrs - : scale == timeScale::Min ? timeInMin : timeInSec; - Logger::testLog<< "Starting Test" << Logger::endl; - size_t num_ort_exception{0}; - size_t num_std_exception{0}; - size_t num_unknown_exception{0}; - size_t num_successful_runs{0}; - size_t iteration{0}; - while(currTime < endTime) - { - try - { - onnx::ModelProto bad_model = model_proto; - Logger::testLog<< "Starting Test iteration: " << iteration << Logger::endl; - mutateModelTest(bad_model, mutateModelDirName, userOpt); - num_successful_runs++; - Logger::testLog<< "Completed Test iteration: " << iteration++ << Logger::endl; - } - catch(const Ort::Exception& ortException) - { - num_ort_exception++; - Logger::testLog<< L"onnx runtime exception: " << ortException.what() << Logger::endl; - Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl; - } - catch(const std::exception& e) - { - num_std_exception++; - Logger::testLog<< L"standard exception: " << e.what() << Logger::endl; - Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl; - } - catch(...) - { - num_unknown_exception++; - Logger::testLog<< L"unknown exception: " << Logger::endl; - Logger::testLog<< "Failed Test iteration: " << iteration++ << Logger::endl; - throw; - } - - // Update current time + // Initialize the model + // + if (model_proto.ParseFromIstream(&modelStream)) { + if (repo_mode) { + Logger::testLog << L"Running Prediction for: " << model_file_name + << L" with seed " << seed << Logger::endl; + mutateModelTest(model_proto, mutate_model_dir_name, user_opt, env, seed); + Logger::testLog << L"Finished Prediction for: " << model_file_name + << L" with seed " << seed << Logger::endl; + return 0; + } else { + // Call the mutateModelTest // - currTime = std::chrono::system_clock::now(); - } - Logger::testLog<< "Ending Test" << Logger::endl; - - if(userOpt.stress) - { - Logger::testLog.enable(); - } + std::chrono::system_clock::time_point curr_time{std::chrono::system_clock::now()}; - Logger::testLog<< L"Total number of exceptions: " << num_unknown_exception+num_std_exception+num_ort_exception << Logger::endl; - Logger::testLog<< L"Number of Unknown exceptions: " << num_unknown_exception << Logger::endl; - Logger::testLog<< L"Number of ort exceptions: " << num_ort_exception << Logger::endl; - Logger::testLog<< L"Number of std exceptions: " << num_std_exception << Logger::endl; - Logger::testLog << L"Number of unique errors: "<< werrStreamBuf.get_unique_errors() << L"\n"; - - if(userOpt.stress) - { - Logger::testLog.disable(); - Logger::testLog.flush(); + std::chrono::minutes time_in_min{test_time_out}; + std::chrono::seconds time_in_sec{test_time_out}; + std::chrono::hours time_in_hrs{test_time_out}; + std::chrono::system_clock::time_point end_time{curr_time}; + end_time += scale == timeScale::Hrs ? time_in_hrs + : scale == timeScale::Min ? time_in_min : time_in_sec; + Logger::testLog << "Starting Test" << Logger::endl; + while (curr_time < end_time) { + try { + onnx::ModelProto bad_model = model_proto; + Logger::testLog << "Starting Test iteration: " << run_stats.iteration << Logger::endl; + mutateModelTest(bad_model, mutate_model_dir_name, user_opt, env); + run_stats.num_successful_runs++; + Logger::testLog << "Completed Test iteration: " << run_stats.iteration++ << Logger::endl; + } catch (...) { + fuzz_handle_exception(run_stats); + } + // Update current time + // + curr_time = std::chrono::system_clock::now(); + } } + } else { + throw std::exception("Unable to initialize the Onnx model in memory"); + } + } else { + std::wstring ort_model_file = model_file; + if (model_file.substr(model_file.find_last_of(L".") + 1) == L"onnx") { + ort_model_file = model_file + L".ort"; + Ort::SessionOptions so; + so.SetGraphOptimizationLevel(ORT_DISABLE_ALL); + so.SetOptimizedModelFilePath(ort_model_file.c_str()); + so.AddConfigEntry(kOrtSessionOptionsConfigSaveModelFormat, "ORT"); + Ort::Session session(env, model_file.c_str(), so); + } else if (model_file.substr(model_file.find_last_of(L".") + 1) != L"ort") { + Logger::testLog << L"Input file name extension is not 'onnx' or 'ort' " << Logger::endl; + return 1; + } + size_t num_bytes = std::filesystem::file_size(ort_model_file); + std::vector model_data(num_bytes); + std::ifstream ortModelStream(ort_model_file, std::ifstream::in | std::ifstream::binary); + ortModelStream.read(model_data.data(), num_bytes); + ortModelStream.close(); + // Currently mutations are generated by using XOR of a byte with the preceeding byte at a time. + // Other possible ways may be considered in future, for example swaping two bytes randomly at a time. + Logger::testLog << "Starting Test" << Logger::endl; + for (size_t& i = run_stats.iteration; i < num_bytes - 1; i++) { + char tmp = model_data[i]; + model_data[i] ^= model_data[i + 1]; + try { + Logger::testLog << "Starting Test iteration: " << i << Logger::endl; + OnnxPrediction predict(model_data, env); + predict.SetupInput(GenerateDataForInputTypeTensor, 0); + predict.RunInference(); + run_stats.num_successful_runs++; + Logger::testLog << "Completed Test iteration: " << i << Logger::endl; + } catch (...) { + fuzz_handle_exception(run_stats); + } + model_data[i] = tmp; } } - else - { - throw std::exception("Unable to initialize the Onnx model in memory"); - } + Logger::testLog << "Ending Test" << Logger::endl; + if (user_opt.stress) { + Logger::testLog.enable(); + } + size_t toal_num_exception = run_stats.num_unknown_exception + run_stats.num_std_exception + run_stats.num_ort_exception; + Logger::testLog << L"Total number of exceptions: " << toal_num_exception << Logger::endl; + Logger::testLog << L"Number of Unknown exceptions: " << run_stats.num_unknown_exception << Logger::endl; + Logger::testLog << L"Number of ort exceptions: " << run_stats.num_ort_exception << Logger::endl; + Logger::testLog << L"Number of std exceptions: " << run_stats.num_std_exception << Logger::endl; + Logger::testLog << L"Number of unique errors: " << werr_stream_buf.get_unique_errors() << L"\n"; + + if (user_opt.stress) { + Logger::testLog.disable(); + Logger::testLog.flush(); + } return 0; - } - catch(const Ort::Exception& ortException) - { - Logger::testLog<< L"onnx runtime exception: " << ortException.what() << Logger::endl; - } - catch(const std::exception& e) - { - Logger::testLog<< L"standard exception: " << e.what() << Logger::endl; - } - catch(...) - { - Logger::testLog<< L"Something Went very wrong: " << Logger::endl; + } catch (const Ort::Exception& ort_exception) { + Logger::testLog << L"onnx runtime exception: " << ort_exception.what() << Logger::endl; + } catch (const std::exception& e) { + Logger::testLog << L"standard exception: " << e.what() << Logger::endl; + } catch (...) { + Logger::testLog << L"Something Went very wrong: " << Logger::endl; } return 1; diff --git a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml new file mode 100644 index 0000000000..8a83568b27 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml @@ -0,0 +1,157 @@ +jobs: +- job: 'build' + pool: 'Win-CPU-2019' + strategy: + maxParallel: 2 + matrix: + debug: + BuildConfig: 'Debug' + release: + BuildConfig: 'RelWithDebInfo' + variables: + OrtPackageId: 'Microsoft.ML.OnnxRuntime' + MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + EnvSetupScript: setup_env.bat + buildArch: x64 + setVcvars: true + ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' + timeoutInMinutes: 120 + workspace: + clean: all + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) + + - task: NodeTool@0 + inputs: + versionSpec: '12.x' + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy flake8 + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" + python setup.py bdist_wheel + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 5.7.0 + inputs: + versionSpec: 5.7.0 + + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + command: 'restore' + feedsToUse: 'config' + restoreSolution: '$(Build.SourcesDirectory)\packages.config' + nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config' + restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)' + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--gen_doc --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_dnnl --use_winml --use_openmp --build_shared_lib --enable_onnx_tests --enable_wcos --build_java --build_nodejs --use_full_protobuf --fuzz_testing' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' + platform: 'x64' + configuration: $(BuildConfig) + msbuildArgs: $(MsbuildArguments) + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: false + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + createLogFile: true + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel' + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + + - template: templates/set-test-data-variables-step.yml + + - task: DotNetCoreCLI@2 + displayName: 'Restore nuget packages' + inputs: + command: restore + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Build C#' + inputs: + command: build + projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: DotNetCoreCLI@2 + displayName: 'Test C#' + condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) + inputs: + command: test + projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - script: | + mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models + DIR dist\ /S /B > wheel_filename_file + set /p WHEEL_FILENAME=