diff --git a/README.md b/README.md index 784382c1f1..3f5b4868c2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ ONNX Runtime is up to date and backwards compatible with all operators (both DNN ### Binaries -Official builds are available on PyPi (Python) and Nuget (C#/C/C++): +Official builds are available on PyPi (Python), Nuget (C#/C/C++), Maven Central (Java), and npm (node.js). * Default CPU Provider (Eigen + MLAS) * GPU Provider - NVIDIA CUDA @@ -78,9 +78,14 @@ Official builds are available on PyPi (Python) and Nuget (C#/C/C++): Dev builds created from the master branch are available for testing newer changes between official releases. Please use these at your own risk. We strongly advise against deploying these to production workloads as support is limited for dev builds. -|Pypi (Python)|Nuget (C#/C/C++)|Other package repositories| -|---|---|---| -*If using pip, run `pip install --upgrade pip` prior to downloading.*

CPU: [**onnxruntime**](https://pypi.org/project/onnxruntime) / [ort-nightly (dev)](https://test.pypi.org/project/ort-nightly)

GPU: [**onnxruntime-gpu**](https://pypi.org/project/onnxruntime-gpu) / [ort-gpu-nightly (dev)](https://test.pypi.org/project/ort-gpu-nightly) | CPU: [**Microsoft.ML.OnnxRuntime**](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime) / [ort-nightly (dev)](https://aiinfra.visualstudio.com/PublicPackages/_packaging?_a=feed&feed=ORT-Nightly)

GPU: [**Microsoft.ML.OnnxRuntime.Gpu**](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.gpu) / [ort-nightly (dev)](https://aiinfra.visualstudio.com/PublicPackages/_packaging?_a=feed&feed=ORT-Nightly)|[Contributed non-official packages](https://docs.microsoft.com/en-us/windows/ai/windows-ml/get-started-uwp) (including Homebrew, Linuxbrew, and nixpkgs)

*These are not maintained by the core ONNX Runtime team and may have limited support; use at your discretion.*| +|Repository|Details| +|---|---| +|Pypi (Python)|*If using pip, run `pip install --upgrade pip` prior to downloading.*
CPU: [**onnxruntime**](https://pypi.org/project/onnxruntime) / [ort-nightly (dev)](https://test.pypi.org/project/ort-nightly)
GPU: [**onnxruntime-gpu**](https://pypi.org/project/onnxruntime-gpu) / [ort-gpu-nightly (dev)](https://test.pypi.org/project/ort-gpu-nightly)| +|Nuget (C#/C/C++)|CPU: [**Microsoft.ML.OnnxRuntime**](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime) / [ort-nightly (dev)](https://aiinfra.visualstudio.com/PublicPackages/_packaging?_a=feed&feed=ORT-Nightly)
GPU: [**Microsoft.ML.OnnxRuntime.Gpu**](https://www.nuget.org/packages/Microsoft.ML.OnnxRuntime.gpu) / [ort-nightly (dev)](https://aiinfra.visualstudio.com/PublicPackages/_packaging?_a=feed&feed=ORT-Nightly)| +|Maven Central (Java)|CPU: [**com.microsoft.onnxruntime/onnxruntime**](https://search.maven.org/artifact/com.microsoft.onnxruntime/onnxruntime)
GPU: [**com.microsoft.onnxruntime/onnxruntime_gpu**](https://search.maven.org/artifact/com.microsoft.onnxruntime/onnxruntime_gpu)| +|npm (node.js)|CPU: [**onnxruntime**](https://www.npmjs.com/package/onnxruntime)| +|Other|[Contributed non-official packages](https://docs.microsoft.com/en-us/windows/ai/windows-ml/get-started-uwp) (including Homebrew, Linuxbrew, and nixpkgs)
*These are not maintained by the core ONNX Runtime team and may have limited support; use at your discretion.*| + #### System Requirements @@ -137,7 +142,7 @@ For production scenarios, it's strongly recommended to build only from an [offic |CPU|GPU|IoT/Edge/Mobile|Other| |---|---|---|---| -||||| +||||| * [Roadmap: Upcoming accelerators](./docs/Roadmap.md#accelerators-and-execution-providers) * [Extensibility: Add an execution provider](docs/AddingExecutionProvider.md) diff --git a/onnxruntime/core/optimizer/insert_cast_transformer.cc b/onnxruntime/core/optimizer/insert_cast_transformer.cc index 069bc0678b..d2781bcbea 100644 --- a/onnxruntime/core/optimizer/insert_cast_transformer.cc +++ b/onnxruntime/core/optimizer/insert_cast_transformer.cc @@ -254,7 +254,10 @@ Status InsertCastTransformer::ApplyImpl(onnxruntime::Graph& graph, bool& modifie ORT_ENFORCE(dtype_attribute->second.has_i(), "InsertCastTransformer works on the assumption that `dtype` attribute holds an integer."); - dtype_attribute->second.set_i(TensorProto_DataType_FLOAT); + // Modify the dtype attribute (which defines the output type) to FLOAT if it is FLOAT16. + if (dtype_attribute->second.i() == TensorProto_DataType_FLOAT16) { + dtype_attribute->second.set_i(TensorProto_DataType_FLOAT); + } } } diff --git a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc index 5fee182fb1..21bb96b631 100644 --- a/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc +++ b/onnxruntime/core/providers/cuda/reduction/reduction_ops.cc @@ -395,10 +395,12 @@ Status ReduceComputeCore(CUDAExecutionProvider& cuda_ep, const Tensor& input, Pr } CudnnReduceDescriptor reduce_desc; - if (std::is_same::value) + if (std::is_same::value) { ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, CudnnTensor::GetDataType(), ReduceTensorIndices)); - else + } else { ORT_RETURN_IF_ERROR(reduce_desc.Set(cudnn_reduce_op, cudnn_type_X, ReduceTensorIndices)); + } + const auto one = Consts::One; const auto zero = Consts::Zero; CudnnTensor input_tensor; @@ -437,7 +439,11 @@ Status ReduceComputeCore(CUDAExecutionProvider& cuda_ep, const Tensor& input, Pr } else { // Reduce max -- Max/Min will output indices data CudnnReduceDescriptor reduce_max_desc; - ORT_RETURN_IF_ERROR(reduce_max_desc.Set(CUDNN_REDUCE_TENSOR_MAX, cudnn_type_X, CUDNN_REDUCE_TENSOR_NO_INDICES)); + cudnnDataType_t cudnn_reduce_max_type = cudnn_type_X; + if((std::is_same::value)) { + cudnn_reduce_max_type = CUDNN_DATA_FLOAT; + } + ORT_RETURN_IF_ERROR(reduce_max_desc.Set(CUDNN_REDUCE_TENSOR_MAX, cudnn_reduce_max_type, CUDNN_REDUCE_TENSOR_NO_INDICES)); size_t indices_bytes_max = 0; CUDNN_RETURN_IF_ERROR(cudnnGetReductionIndicesSize(cuda_ep.PerThreadCudnnHandle(), reduce_max_desc, input_tensor, output_tensor, &indices_bytes_max)); diff --git a/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb index ea7b5f7ab8..caba3614fa 100644 --- a/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -54,11 +54,11 @@ "source": [ "import sys\n", " \n", - "!{sys.executable} -m pip install --quiet --upgrade tensorflow==2.2.0\n", - "!{sys.executable} -m pip install --quiet --upgrade onnxruntime\n", - "!{sys.executable} -m pip install --quiet --upgrade onnxruntime-tools\n", - "!{sys.executable} -m pip install --quiet --upgrade keras2onnx\n", - "!{sys.executable} -m pip install --quiet transformers==2.11.0\n", + "!{sys.executable} -m pip install --quiet --upgrade tensorflow==2.3.0\n", + "!{sys.executable} -m pip install --quiet --upgrade onnxruntime==1.4.0\n", + "!{sys.executable} -m pip install --quiet --upgrade onnxruntime-tools==1.4.0\n", + "!{sys.executable} -m pip install --quiet --upgrade keras2onnx==1.7.0\n", + "!{sys.executable} -m pip install --quiet transformers==3.0.2\n", "!{sys.executable} -m pip install --quiet wget pandas" ] }, @@ -92,13 +92,23 @@ "outputs": [], "source": [ "import os\n", - "cache_dir = './cached_models'\n", + "cache_dir = './cache_models'\n", "output_dir = './onnx_models'\n", "for directory in [cache_dir, output_dir]:\n", " if not os.path.exists(directory):\n", " os.makedirs(directory)" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "tf.config.set_visible_devices([], 'GPU') # Disable GPU for fair comparison" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -115,16 +125,29 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at bert-base-cased were not used when initializing TFBertForQuestionAnswering: ['nsp___cls', 'mlm___cls']\n", + "- This IS expected if you are initializing TFBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n", + "- This IS NOT expected if you are initializing TFBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of TFBertForQuestionAnswering were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['qa_outputs']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], "source": [ "from transformers import (TFBertForQuestionAnswering, BertTokenizer)\n", "\n", "#model_name_or_path = 'bert-large-uncased-whole-word-masking-finetuned-squad'\n", "model_name_or_path = \"bert-base-cased\"\n", + "is_fine_tuned = (model_name_or_path == 'bert-large-uncased-whole-word-masking-finetuned-squad')\n", "\n", "# Load model and tokenizer\n", "tokenizer = BertTokenizer.from_pretrained(model_name_or_path, do_lower_case=True, cache_dir=cache_dir)\n", @@ -144,42 +167,34 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The answer is: [CLS] what is on ##nx run ##time ? [SEP] on ##nx run ##time is a performance - focused in ##ference engine for on ##nx models\n" - ] - } - ], + "outputs": [], "source": [ - "import tensorflow as tf\n", "import numpy\n", "\n", "question, text = \"What is ONNX Runtime?\", \"ONNX Runtime is a performance-focused inference engine for ONNX models.\"\n", "# Pad to max length is needed. Otherwise, position embedding might be truncated by constant folding.\n", "inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors='tf',\n", - " max_length=max_sequence_length, pad_to_max_length=True)\n", + " max_length=max_sequence_length, pad_to_max_length=True, truncation=True)\n", "start_scores, end_scores = model(inputs)\n", "\n", "num_tokens = len(inputs[\"input_ids\"][0])\n", - "all_tokens = tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0])\n", - "print(\"The answer is:\", ' '.join(all_tokens[numpy.argmax(start_scores) : numpy.argmax(end_scores)+1]))" + "if is_fine_tuned:\n", + " all_tokens = tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0])\n", + " print(\"The answer is:\", ' '.join(all_tokens[numpy.argmax(start_scores) : numpy.argmax(end_scores)+1]))" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Tensorflow Inference time for sequence length 512 = 94.62 ms\n" + "Tensorflow Inference time for sequence length 512 = 1133.13 ms\n" ] } ], @@ -203,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -239,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -260,14 +275,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ONNX Runtime cpu inference time for sequence length 512 (model not optimized): 630.54 ms\n" + "ONNX Runtime cpu inference time for sequence length 512 (model not optimized): 654.49 ms\n" ] } ], @@ -280,7 +295,7 @@ "\n", "# intra_op_num_threads=1 can be used to enable OpenMP in OnnxRuntime 1.2.0.\n", "# For OnnxRuntime 1.3.0 or later, this does not have effect unless you are using onnxruntime-gpu package.\n", - "sess_options.intra_op_num_threads=1\n", + "# sess_options.intra_op_num_threads=1\n", "\n", "# Providers is optional. Only needed when you use onnxruntime-gpu for CPU inference.\n", "session = onnxruntime.InferenceSession(output_model_path, sess_options, providers=['CPUExecutionProvider'])\n", @@ -302,26 +317,15 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "***** Verifying correctness (TensorFlow and ONNX Runtime) *****\n", - "WARNING:tensorflow:From :2: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.identity instead.\n", - "start_scores are close: False\n", - "end_scores are close: False\n" - ] - } - ], + "outputs": [], "source": [ - "print(\"***** Verifying correctness (TensorFlow and ONNX Runtime) *****\")\n", - "print('start_scores are close:', numpy.allclose(results[0], start_scores.cpu(), rtol=1e-05, atol=1e-04))\n", - "print('end_scores are close:', numpy.allclose(results[1], end_scores.cpu(), rtol=1e-05, atol=1e-04))" + "# Some weights of TFBertForQuestionAnswering might not be initialized without fine-tuning.\n", + "if is_fine_tuned:\n", + " print(\"***** Verifying correctness (TensorFlow and ONNX Runtime) *****\")\n", + " print('start_scores are close:', numpy.allclose(results[0], start_scores.cpu(), rtol=1e-05, atol=1e-04))\n", + " print('end_scores are close:', numpy.allclose(results[1], end_scores.cpu(), rtol=1e-05, atol=1e-04))" ] }, { @@ -346,7 +350,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -367,14 +371,14 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ONNX Runtime cpu inference time on optimized model: 369.18 ms\n" + "ONNX Runtime cpu inference time on optimized model: 328.48 ms\n" ] } ], @@ -394,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -402,15 +406,15 @@ "output_type": "stream", "text": [ "***** Verifying correctness (before and after optimization) *****\n", - "start_scores are close: False\n", - "end_scores are close: False\n" + "start_scores are close: True\n", + "end_scores are close: True\n" ] } ], "source": [ "print(\"***** Verifying correctness (before and after optimization) *****\")\n", - "print('start_scores are close:', numpy.allclose(opt_results[0], start_scores.cpu(), rtol=1e-05, atol=1e-04))\n", - "print('end_scores are close:', numpy.allclose(opt_results[1], end_scores.cpu(), rtol=1e-05, atol=1e-04))" + "print('start_scores are close:', numpy.allclose(opt_results[0], results[0], rtol=1e-05, atol=1e-04))\n", + "print('end_scores are close:', numpy.allclose(opt_results[1], results[1], rtol=1e-05, atol=1e-04))" ] }, { @@ -426,7 +430,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -434,8 +438,8 @@ "output_type": "stream", "text": [ "100% passed for 10 random inputs given thresholds (rtol=0.001, atol=0.0001).\n", - "maximum absolute difference=1.2461096048355103e-06\n", - "maximum relative difference=0.006510902661830187\n" + "maximum absolute difference=1.6242265701293945e-06\n", + "maximum relative difference=0.009154098108410835\n" ] } ], @@ -457,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -465,17 +469,17 @@ "output_type": "stream", "text": [ "Running test: model=keras_bert-base-cased_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n", - "Average latency = 99.01 ms, Throughput = 10.10 QPS\n", + "Average latency = 97.93 ms, Throughput = 10.21 QPS\n", "test setting TestSetting(batch_size=1, sequence_length=128, test_cases=100, test_times=1, contiguous=None, use_gpu=False, warmup=True, omp_num_threads=12, omp_wait_policy='ACTIVE', intra_op_num_threads=1, seed=3, verbose=False, inclusive=False, extra_latency=True)\n", "Generating 100 samples for batch_size=1 sequence_length=128\n", - "Test summary is saved to onnx_models\\perf_results_CPU_B1_S128_20200617-210258.txt\n" + "Test summary is saved to onnx_models\\perf_results_CPU_B1_S128_20200728-165907.txt\n" ] } ], "source": [ "THREAD_SETTING = '--intra_op_num_threads 1 --omp_num_threads {} --omp_wait_policy ACTIVE'.format(psutil.cpu_count(logical=True))\n", "\n", - "!{sys.executable} -m onnxruntime_tools.transformers.bert_perf_test --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive $THREAD_SETTING\n" + "!{sys.executable} -m onnxruntime_tools.transformers.bert_perf_test --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive $THREAD_SETTING" ] }, { @@ -487,14 +491,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "./onnx_models\\perf_results_CPU_B1_S128_20200617-210258.txt\n" + "./onnx_models\\perf_results_CPU_B1_S128_20200728-165907.txt\n" ] }, { @@ -534,9 +538,9 @@ " 12\n", " ACTIVE\n", " None\n", - " 99.01\n", - " 130.11\n", - " 10.1\n", + " 97.93\n", + " 158.16\n", + " 10.21\n", " \n", " \n", "\n", @@ -547,10 +551,10 @@ "0 1 12 ACTIVE None \n", "\n", " Latency(ms) Latency_P99 Throughput(QPS) \n", - "0 99.01 130.11 10.1 " + "0 97.93 158.16 10.21 " ] }, - "execution_count": 16, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -589,7 +593,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -602,7 +606,7 @@ " \"devices\": [\n", " {\n", " \"memory_total\": 8589934592,\n", - " \"memory_available\": 1643134976,\n", + " \"memory_available\": 8480882688,\n", " \"name\": \"GeForce GTX 1070\"\n", " }\n", " ]\n", @@ -618,12 +622,12 @@ " },\n", " \"memory\": {\n", " \"total\": 16971259904,\n", - " \"available\": 3282817024\n", + " \"available\": 3480842240\n", " },\n", " \"python\": \"3.6.10.final.0 (64 bit)\",\n", " \"os\": \"Windows-10-10.0.18362-SP0\",\n", " \"onnxruntime\": {\n", - " \"version\": \"1.3.0\",\n", + " \"version\": \"1.4.0\",\n", " \"support_gpu\": false\n", " },\n", " \"pytorch\": {\n", @@ -631,8 +635,8 @@ " \"support_gpu\": false\n", " },\n", " \"tensorflow\": {\n", - " \"version\": \"2.2.0\",\n", - " \"git_version\": \"v2.2.0-rc4-8-g2b96f3662b\",\n", + " \"version\": \"2.3.0\",\n", + " \"git_version\": \"v2.3.0-rc2-23-gb36436b087\",\n", " \"support_gpu\": true\n", " }\n", "}\n" @@ -642,7 +646,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2020-06-17 21:03:03.409601: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll\n" + "2020-07-28 16:59:18.638897: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library cudart64_101.dll\n" ] } ], diff --git a/onnxruntime/test/framework/insert_cast_transformer_test.cc b/onnxruntime/test/framework/insert_cast_transformer_test.cc index 152a407e85..e493784fa5 100644 --- a/onnxruntime/test/framework/insert_cast_transformer_test.cc +++ b/onnxruntime/test/framework/insert_cast_transformer_test.cc @@ -134,8 +134,9 @@ TEST(TransformerTest, ThreeInARowRemoval) { ASSERT_TRUE(op_to_count["Cast"] == 2); } -// test a case where the ONNX inferred type (float16) is different from the type bound -// to the output NodeArg of the "RandomNormalLike" node because of the InsertCaseTransformer +// test a case where the ONNX inferred output type (float16) is different from the type bound +// to the output NodeArg of the "RandomNormalLike" node (input is float16) because of the InsertCaseTransformer +// Here the ONNX inferred output type (float16) must be made float because that is what the kernel produces TEST(TransformerTest, RandomNormalLikeWithFloat16Inputs) { auto model_uri = MODEL_FOLDER ORT_TSTR("random_normal_like_float16.onnx"); std::shared_ptr model; @@ -153,5 +154,25 @@ TEST(TransformerTest, RandomNormalLikeWithFloat16Inputs) { EXPECT_TRUE(status.IsOK()) << status; } +// A case where the ONNX inferred output type is int32 to a node that consumes float16 input +// Here the InsertCastTransformer must not change the ONNX inferred output type and keep it +// as is (int32) +TEST(TransformerTest, MultinomialWithFloat16Input) { + auto model_uri = MODEL_FOLDER ORT_TSTR("multinomial_float16.onnx"); + std::shared_ptr model; + auto status = Model::Load(model_uri, model, nullptr, DefaultLoggingManager().DefaultLogger()); + ASSERT_TRUE(status.IsOK()) << status; + + Graph& graph = model->MainGraph(); + InsertCastTransformer transformer("Test"); + + bool modified = false; + status = transformer.Apply(graph, modified, DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(status.IsOK()) << status; + EXPECT_TRUE(modified) << "Transformer should have added some Cast nodes"; + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status; +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 3864461149..63540c28c6 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -39,6 +39,7 @@ void usage() { "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'dnnl', 'tensorrt', 'ngraph', " "'openvino', 'nuphar', 'migraphx', 'acl' or 'armnn'. " "Default: 'cpu'.\n" + "\t-p: Pause after launch, can attach debugger and continue\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-d [device_id]: Specifies the device id for multi-device (e.g. GPU). The value should > 0\n" "\t-o [optimization level]: Default is 99. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" @@ -109,9 +110,10 @@ int real_main(int argc, char* argv[], Ort::Env& env) { int verbosity_option_count = 0; OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_ERROR; + bool pause = false; { int ch; - while ((ch = getopt(argc, argv, ORT_TSTR("Ac:hj:Mn:r:e:xvo:d:"))) != -1) { + while ((ch = getopt(argc, argv, ORT_TSTR("Ac:hj:Mn:r:e:xvo:d:p"))) != -1) { switch (ch) { case 'A': enable_cpu_mem_arena = false; @@ -181,6 +183,9 @@ int real_main(int argc, char* argv[], Ort::Env& env) { case 'x': execution_mode = ExecutionMode::ORT_PARALLEL; break; + case 'p': + pause = true; + break; case 'o': { int tmp = static_cast(OrtStrtol(optarg, nullptr)); switch (tmp) { @@ -243,6 +248,12 @@ int real_main(int argc, char* argv[], Ort::Env& env) { return -1; } + if (pause) { + printf("Enter to continue...\n"); + fflush(stdout); + getchar(); + } + try { env = Ort::Env{logging_level, "Default"}; } catch (std::exception& ex) { diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 624170d73c..6ad7b32f4d 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -58,7 +58,7 @@ TEST_P(ModelTest, Run) { // them is enabled here to save CI build time. return; } - if (model_info->GetONNXOpSetVersion() != 12 && provider_name == "dnnl") { + if (model_info->GetONNXOpSetVersion() == 10 && provider_name == "dnnl") { // DNNL can run most of the model tests, but only part of // them is enabled here to save CI build time. return; diff --git a/onnxruntime/test/testdata/transform/multinomial_float16.onnx b/onnxruntime/test/testdata/transform/multinomial_float16.onnx new file mode 100644 index 0000000000..b72e2e21a2 Binary files /dev/null and b/onnxruntime/test/testdata/transform/multinomial_float16.onnx differ diff --git a/orttraining/orttraining/core/graph/gradient_builder.cc b/orttraining/orttraining/core/graph/gradient_builder.cc index eb6ea80457..48b9635f43 100644 --- a/orttraining/orttraining/core/graph/gradient_builder.cc +++ b/orttraining/orttraining/core/graph/gradient_builder.cc @@ -905,6 +905,40 @@ IMPLEMENT_GRADIENT_BUILDER(GetReduceMeanGradient) { return result; } +// Reference computation is pytorch's logsumexp_backward +// dx_i = exp(xi) / reduceSum(exp(xi)) +// O(0) = log(reduceSum(exp(xi))) +// Self_Sub_Result = I(0) - O(0) = xi - log(sum(exp(xi))) = log( xi / reduceSum(exp(xi))) +// Gradient computation is re-using output and input from forward op, can be a recomputation candidate. +IMPLEMENT_GRADIENT_BUILDER(GetReduceLogSumExpGradient) { + std::vector result; + auto attributes = SrcNodeAttributes(); + bool keepdims = true; + if (attributes.find("keepdims") != attributes.end() && + attributes.at("keepdims").has_i()) { + keepdims = static_cast(attributes.at("keepdims").i()); + } + + ArgDef grad = GO(0); + if (!keepdims && attributes.find("axes") != attributes.end()) { + std::vector axes_values = RetrieveValues(attributes.at("axes")); + grad = IA("Unsqueezed_Grad"); + result.push_back(NodeDef("Unsqueeze", {GO(0)}, {grad}, {MakeAttribute("axes", axes_values)})); + + result.push_back(NodeDef("Unsqueeze", {O(0)}, {IA("Unsqueezed_Output")}, {MakeAttribute("axes", axes_values)})); + result.push_back(NodeDef("Sub", {I(0), IA("Unsqueezed_Output")}, {IA("Self_Sub_Result")})); + } + else { + result.push_back(NodeDef("Sub", {I(0), O(0)}, {IA("Self_Sub_Result")})); + } + + result.push_back(NodeDef("Exp", {IA("Self_Sub_Result")}, {IA("Self_Sub_Result_Exp")})); + + result.push_back(NodeDef("Mul", {IA("Self_Sub_Result_Exp"), grad}, {GI(0)})); + + return result; +} + IMPLEMENT_GRADIENT_BUILDER(GetReduceSumGradient) { std::vector result; auto attributes = SrcNodeAttributes(); diff --git a/orttraining/orttraining/core/graph/gradient_builder.h b/orttraining/orttraining/core/graph/gradient_builder.h index 9a32e421bf..819c800820 100644 --- a/orttraining/orttraining/core/graph/gradient_builder.h +++ b/orttraining/orttraining/core/graph/gradient_builder.h @@ -25,6 +25,7 @@ DECLARE_GRADIENT_BUILDER(GetMulGradient) DECLARE_GRADIENT_BUILDER(GetDivGradient) DECLARE_GRADIENT_BUILDER(GetReduceMeanGradient) DECLARE_GRADIENT_BUILDER(GetReduceSumGradient) +DECLARE_GRADIENT_BUILDER(GetReduceLogSumExpGradient) DECLARE_GRADIENT_BUILDER(GetPowGradient) DECLARE_GRADIENT_BUILDER(GetConcatGradient) DECLARE_GRADIENT_BUILDER(GetReshapeGradient) diff --git a/orttraining/orttraining/core/graph/gradient_builder_registry.cc b/orttraining/orttraining/core/graph/gradient_builder_registry.cc index 94b4e1e096..7631ce25eb 100644 --- a/orttraining/orttraining/core/graph/gradient_builder_registry.cc +++ b/orttraining/orttraining/core/graph/gradient_builder_registry.cc @@ -51,6 +51,7 @@ void GradientBuilderRegistry::RegisterGradientBuilders() { REGISTER_GRADIENT_BUILDER("Pow", GetPowGradient); REGISTER_GRADIENT_BUILDER("ReduceMean", GetReduceMeanGradient); REGISTER_GRADIENT_BUILDER("ReduceSum", GetReduceSumGradient); + REGISTER_GRADIENT_BUILDER("ReduceLogSumExp", GetReduceLogSumExpGradient); REGISTER_GRADIENT_BUILDER("Add", GetAddSubGradient); REGISTER_GRADIENT_BUILDER("Sub", GetAddSubGradient); REGISTER_GRADIENT_BUILDER("Mul", GetMulGradient); diff --git a/orttraining/orttraining/python/ort_trainer.py b/orttraining/orttraining/python/ort_trainer.py index a69c41e312..b350e3c578 100644 --- a/orttraining/orttraining/python/ort_trainer.py +++ b/orttraining/orttraining/python/ort_trainer.py @@ -629,6 +629,8 @@ class ORTTrainer(): self.world_size = world_size self.use_mixed_precision = use_mixed_precision + self.original_model_state_keys = list(model.state_dict().keys()) if hasattr(model, 'state_dict') else [] + self.session = None self.device_ = device self.gradient_accumulation_steps = gradient_accumulation_steps @@ -773,7 +775,11 @@ class ORTTrainer(): if n.name not in torch_state: torch_state[n.name] = torch.from_numpy(numpy_helper.to_array(n)) - return torch_state + # Need to remove redundant initializers and name suffices to map back to original torch state names + torch_state_to_return = {key: torch_state[key] for key in self.original_model_state_keys if key in torch_state} \ + if self.original_model_state_keys \ + else torch_state + return torch_state_to_return def load_state_dict(self, state_dict, strict=False): # Note: It may happen ONNX model has not yet been initialized diff --git a/orttraining/orttraining/test/gradient/gradient_op_test_utils.h b/orttraining/orttraining/test/gradient/gradient_op_test_utils.h index ad75d06162..26f56ddbe0 100644 --- a/orttraining/orttraining/test/gradient/gradient_op_test_utils.h +++ b/orttraining/orttraining/test/gradient/gradient_op_test_utils.h @@ -7,6 +7,9 @@ namespace onnxruntime { namespace test { +using TestDataVector = std::tuple>, // Input data + std::vector>, // output data + std::vector>>; //attribute class GradientOpTester : public OpTester { public: @@ -39,3 +42,4 @@ class GradientOpTester : public OpTester { }; } // namespace test } // namespace onnxruntime + diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc index b5f43f263f..03095320c3 100644 --- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc +++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc @@ -38,6 +38,70 @@ static bool IsErrorWithinTolerance(float error, float tolerance) { #define EXPECT_IS_TINY(max_error) \ EXPECT_IS_TINIER_THAN(max_error, 1.5e-2f) +static void RunReductionTests(const OpDef& op_def) { + + TestDataVector test_data( + // Input X + { + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + {{4, 3, 2}}, + }, + // Input Y + { + {{1, 1, 1}}, + {{}}, + {{1, 3, 1}}, + {{2}}, + {{4, 1, 2}}, + {{4, 3}}, + {{4, 1, 2}}, + {{4}} + }, + // Attributes + { + // default + {}, + // axes = [0, 1, 2], keepdims = 0 + {MakeAttribute("axes", std::vector{0, 1, 2}), + MakeAttribute("keepdims", int64_t(0))}, + // axes = [0, 2], keepdims = 1 + {MakeAttribute("axes", std::vector{0, 2})}, + // axes = [0, 1], keepdims = 0 + {MakeAttribute("axes", std::vector{0, 1}), + MakeAttribute("keepdims", int64_t(0))}, + // axes = [1], keepdims = 1 + {MakeAttribute("axes", std::vector{1}), + MakeAttribute("keepdims", int64_t(1))}, + // axes = [2], keepdims = 0 + {MakeAttribute("axes", std::vector{2}), + MakeAttribute("keepdims", int64_t(0))}, + // axes = [-2], keepdims = 1 + {MakeAttribute("axes", std::vector{-2}), + MakeAttribute("keepdims", int64_t(1))}, + // axes = [-2, -1], keepdims = 0 + {MakeAttribute("axes", std::vector{-2, -1}), + MakeAttribute("keepdims", int64_t(0))} + }); + + GradientChecker gradient_checker; + + float max_error; + + for (size_t i = 0; i < std::get<0>(test_data).size(); i++) { + max_error = 0; + gradient_checker.ComputeGradientError(op_def, std::get<0>(test_data)[i], + std::get<1>(test_data)[i], &max_error, + std::get<2>(test_data)[i]); + EXPECT_IS_TINY(max_error); + } +} + template void GenerateRandomDataWithOneHot( std::vector>& x_datas, @@ -426,149 +490,24 @@ TEST(GradientCheckerTest, GemmGrad) { } TEST(GradientCheckerTest, ReduceMeanGrad) { - float max_error; - GradientChecker gradient_checker; // Attribute axes supports negative values from opset 11. OpDef op_def{"ReduceMean", kOnnxDomain, 11}; - // default - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{1, 1, 1}}, &max_error); - EXPECT_IS_TINY(max_error); - } - - // TODO: Fix forward kernel behavior for default axes - // default axes, keepdims = 0 - /* - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{}}, &max_error, - {MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - */ - - // axes = [0, 1, 2], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{}}, &max_error, - {MakeAttribute("axes", std::vector{0, 1, 2}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [0, 2], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{1, 3, 1}}, &max_error, - {MakeAttribute("axes", std::vector{0, 2})}); - EXPECT_IS_TINY(max_error); - } - - // axes = [0, 1], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{2}}, &max_error, - {MakeAttribute("axes", std::vector{0, 1}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [1], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 1, 2}}, &max_error, - {MakeAttribute("axes", std::vector{1}), - MakeAttribute("keepdims", int64_t(1))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [2], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 3}}, &max_error, - {MakeAttribute("axes", std::vector{2}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [-2], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 1, 2}}, &max_error, - {MakeAttribute("axes", std::vector{-2}), - MakeAttribute("keepdims", int64_t(1))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [-2, -1], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4}}, &max_error, - {MakeAttribute("axes", std::vector{-2, -1}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } + RunReductionTests(op_def); } TEST(GradientCheckerTest, ReduceSumGrad) { - float max_error; - GradientChecker gradient_checker; // Attribute axes supports negative values from opset 11. OpDef op_def{"ReduceSum", kOnnxDomain, 11}; - // default - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{1, 1, 1}}, &max_error); - EXPECT_IS_TINY(max_error); - } + RunReductionTests(op_def); +} - // axes = [0, 1, 2], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{}}, &max_error, - {MakeAttribute("axes", std::vector{0, 1, 2}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } +TEST(GradientCheckerTest, ReduceLogSumExpGrad) { + // Attribute axes supports negative values from opset 11. + OpDef op_def{"ReduceLogSumExp", kOnnxDomain, 11}; - // axes = [0, 2], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{1, 3, 1}}, &max_error, - {MakeAttribute("axes", std::vector{0, 2})}); - EXPECT_IS_TINY(max_error); - } - - // axes = [0, 1], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{2}}, &max_error, - {MakeAttribute("axes", std::vector{0, 1}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [1], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 1, 2}}, &max_error, - {MakeAttribute("axes", std::vector{1}), - MakeAttribute("keepdims", int64_t(1))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [2], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 3}}, &max_error, - {MakeAttribute("axes", std::vector{2}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [-2], keepdims = 1 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{4, 1, 2}}, &max_error, - {MakeAttribute("axes", std::vector{-2}), - MakeAttribute("keepdims", int64_t(1))}); - EXPECT_IS_TINY(max_error); - } - - // axes = [-1, -3], keepdims = 0 - { - gradient_checker.ComputeGradientError(op_def, {{4, 3, 2}}, {{3}}, &max_error, - {MakeAttribute("axes", std::vector{-1, -3}), - MakeAttribute("keepdims", int64_t(0))}); - EXPECT_IS_TINY(max_error); - } + RunReductionTests(op_def); } #ifndef USE_CUDA @@ -1960,3 +1899,4 @@ TEST(GradientCheckerTest, ExpandGrad) { } // namespace onnxruntime #endif // NDEBUG + diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index e551143ef5..cc21b28368 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -6,4 +6,4 @@ jobs: BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --build_wheel --build_java --build_nodejs --enable_language_interop_ops --use_featurizers --wheel_name_suffix featurizers"' DoNugetPack: 'false' ArtifactName: 'drop-linux' - TimeoutInMinutes: 120 + TimeoutInMinutes: 180 diff --git a/tools/python/ort_test_dir_utils.py b/tools/python/ort_test_dir_utils.py index 993bd1cb41..6dc3fb0217 100644 --- a/tools/python/ort_test_dir_utils.py +++ b/tools/python/ort_test_dir_utils.py @@ -87,7 +87,7 @@ def create_test_dir(model_path, root_path, test_name, if not os.path.exists(test_dir) or not os.path.exists(test_data_dir): os.makedirs(test_data_dir) - model_filename = model_path.split('\\')[-1] + model_filename = os.path.split(model_path)[-1] test_model_filename = os.path.join(test_dir, model_filename) shutil.copy(model_path, test_model_filename) diff --git a/winml/adapter/winml_adapter_dml.cpp b/winml/adapter/winml_adapter_dml.cpp index 1c44c96503..10e412e68b 100644 --- a/winml/adapter/winml_adapter_dml.cpp +++ b/winml/adapter/winml_adapter_dml.cpp @@ -19,10 +19,27 @@ namespace winmla = Windows::AI::MachineLearning::Adapter; #ifdef USE_DML + +EXTERN_C IMAGE_DOS_HEADER __ImageBase; + +static bool IsCurrentModuleInSystem32() { + std::string current_module_path; + current_module_path.reserve(MAX_PATH); + auto size_module_path = GetModuleFileNameA((HINSTANCE)&__ImageBase, current_module_path.data(), MAX_PATH); + FAIL_FAST_IF(size_module_path == 0); + + std::string system32_path; + system32_path.reserve(MAX_PATH); + auto size_system32_path = GetSystemDirectoryA(system32_path.data(), MAX_PATH); + FAIL_FAST_IF(size_system32_path == 0); + + return _strnicmp(system32_path.c_str(), current_module_path.c_str(), size_system32_path) == 0; +} + Microsoft::WRL::ComPtr CreateDmlDevice(ID3D12Device* d3d12Device) { DWORD flags = 0; #ifdef BUILD_INBOX - flags = LOAD_LIBRARY_SEARCH_SYSTEM32; + flags |= IsCurrentModuleInSystem32() ? LOAD_LIBRARY_SEARCH_SYSTEM32 : 0; #endif // Dynamically load DML to avoid WinML taking a static dependency on DirectML.dll diff --git a/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp b/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp index 8940f9642d..f967500bc6 100644 --- a/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp +++ b/winml/lib/Api.Ort/OnnxruntimeEnvironment.cpp @@ -13,10 +13,26 @@ using namespace _winml; static bool debug_output_ = false; +EXTERN_C IMAGE_DOS_HEADER __ImageBase; + +static bool IsCurrentModuleInSystem32() { + std::string current_module_path; + current_module_path.reserve(MAX_PATH); + auto size_module_path = GetModuleFileNameA((HINSTANCE)&__ImageBase, current_module_path.data(), MAX_PATH); + FAIL_FAST_IF(size_module_path == 0); + + std::string system32_path; + system32_path.reserve(MAX_PATH); + auto size_system32_path = GetSystemDirectoryA(system32_path.data(), MAX_PATH); + FAIL_FAST_IF(size_system32_path == 0); + + return _strnicmp(system32_path.c_str(), current_module_path.c_str(), size_system32_path) == 0; +} + static HRESULT GetOnnxruntimeLibrary(HMODULE& module) { DWORD flags = 0; #ifdef BUILD_INBOX - flags = LOAD_LIBRARY_SEARCH_SYSTEM32; + flags |= IsCurrentModuleInSystem32() ? LOAD_LIBRARY_SEARCH_SYSTEM32 : 0; #endif auto out_module = LoadLibraryExA("onnxruntime.dll", nullptr, flags);