diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index f0b681b59a..04ac0cb65a 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -81,11 +81,6 @@ option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir" option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF) option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump node input shapes and output data to standard output when executing the model." OFF) -if(UNIX AND onnxruntime_USE_LLVM) - #prebuilt llmv biniaries use the old GNU C++ ABI - add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) -endif() - set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) #nsync tests failed on Mac Build set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) diff --git a/dockerfiles/Dockerfile.nuphar b/dockerfiles/Dockerfile.nuphar new file mode 100644 index 0000000000..d776f310ee --- /dev/null +++ b/dockerfiles/Dockerfile.nuphar @@ -0,0 +1,29 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +FROM ubuntu:16.04 + +ARG PYTHON_VERSION=3.5 +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_SERVER_BRANCH=master + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y sudo git bash + +ENV PATH="/opt/cmake/bin:${PATH}" +RUN git clone --single-branch --branch ${ONNXRUNTIME_SERVER_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime +RUN /onnxruntime/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && \ + /onnxruntime/tools/ci_build/github/linux/docker/scripts/install_deps.sh + +WORKDIR / + +RUN mkdir -p /onnxruntime/build && \ + pip3 install sympy packaging && \ + python3 /onnxruntime/tools/ci_build/build.py --build_dir /onnxruntime/build --config Release --build_shared_lib --skip_submodule_sync --build_wheel --parallel --use_nuphar --use_mklml --use_tvm --use_llvm + +RUN pip3 install /onnxruntime/build/Release/dist/onnxruntime_nuphar-*.whl && \ + rm -rf /onnxruntime diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 5c54e2e247..8d10f8dada 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -7,6 +7,7 @@ - [TensorRT](Dockerfile.tensorrt) - [OpenVINO](Dockerfile.openvino) - [ONNX Runtime Server](Dockerfile.server) +- [Nuphar](Dockerfile.nuphar) **Preparation step:** download `scripts` to your local folder before running the `docker build` command for any of the options below. @@ -195,3 +196,20 @@ curl -X POST -d "@request.json" -H "Content-Type: application/json" http://0.0.0.0:{your_local_port}/v1/models/mymodel/versions/3:predict ``` +## Nuphar (Public Preview) +#### Linux 16.04, Python Bindings + +1. Build the docker image from the Dockerfile in this repository. + ``` + # If you have a Linux machine, preface this command with "sudo" + + docker build -t onnxruntime-nuphar -f Dockerfile.nuphar . + ``` + +2. Run the Docker image + + ``` + # If you have a Linux machine, preface this command with "sudo" + + docker run -it onnxruntime-nuphar + ``` \ No newline at end of file diff --git a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc index 28c7107755..4482c7fcf7 100644 --- a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc @@ -78,7 +78,7 @@ static std::vector x_vals_3dims = { 1.2940853f, 1.0387882f, 1.7437122f, 0.79806274f, 0.02968323f, 1.0693159f, 0.8907064f, 1.7548862f, 1.4956441f, 1.0693927f}; -TEST(LogSoftmaxOperator, DISABLED_ThreeDimsAxis0) { +TEST(LogSoftmaxOperator, ThreeDimsAxis0) { // x = // node = onnx.helper.make_node('LogSoftmax', inputs = ['x'], outputs = ['y'], axis = 0) // y = logsoftmax_2d(x.reshape(1, 60)).reshape(3, 4, 5) diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc index 4ecf92f61c..0273378194 100644 --- a/onnxruntime/test/providers/cpu/math/softmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -74,7 +74,7 @@ static std::vector x_vals_3dims = { 1.2940853f, 1.0387882f, 1.7437122f, 0.79806274f, 0.02968323f, 1.0693159f, 0.8907064f, 1.7548862f, 1.4956441f, 1.0693927f}; -TEST(SoftmaxOperator, DISABLED_ThreeDimsAxis0) { +TEST(SoftmaxOperator, ThreeDimsAxis0) { // x = // node = onnx.helper.make_node('Softmax', inputs = ['x'], outputs = ['y'], axis = 0) // y = softmax_2d(x.reshape(1, 60)).reshape(3, 4, 5) diff --git a/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc index e3e2feff8a..9bb0a844f0 100644 --- a/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/tensor_op_test.cc @@ -215,7 +215,7 @@ TEST(TensorOpTest, CastToFloat16) { TestCastOp(int64_t_data, float16_output, shape, TensorProto::FLOAT16); } -TEST(TensorOpTest, DISABLED_CastFromFloat16) { +TEST(TensorOpTest, CastFromFloat16) { const std::vector shape{3, 2, 2}; const std::initializer_list float_output = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; const std::initializer_list input = { diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index f1f23f066a..164f598cb8 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -86,8 +86,7 @@ def create_backend_test(testname=None): # Type not supported backend_test.exclude(r'(FLOAT16)') - backend_test.exclude(r'(test_logsoftmax_axis_0)') - backend_test.exclude(r'(test_softmax_axis_0)') + if testname: backend_test.include(testname + '.*') else: @@ -120,6 +119,7 @@ def create_backend_test(testname=None): '^test_top_k*', '^test_unique_*', '^test_mod_float_mixed_sign_example_cpu.*', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types + '^test_shrink_cpu.*', #Invalid rank for input: x Got: 1 Expected: 2 Please fix either the inputs or the model. ) # Example of how to disable tests for a specific provider. diff --git a/onnxruntime/test/python/onnxruntime_test_python_nuphar.py b/onnxruntime/test/python/onnxruntime_test_python_nuphar.py index db534dac6b..a5c1ab44aa 100644 --- a/onnxruntime/test/python/onnxruntime_test_python_nuphar.py +++ b/onnxruntime/test/python/onnxruntime_test_python_nuphar.py @@ -56,20 +56,14 @@ class TestNuphar(unittest.TestCase): nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) - non_jit_repeats = 1 - jit_repeats = 10 - # prepare feed feed = {} for i in range(4): tp = onnx.load_tensor(os.path.join(bidaf_dir, 'test_data_set_0', 'input_{}.pb'.format(i))) feed[tp.name] = numpy_helper.to_array(tp) - start = timer() sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session - for i in range(non_jit_repeats): - output = sess.run([], feed) - non_jit_time = timer() - start + output = sess.run([], feed) cache_dir_content = os.listdir(cache_dir) assert len(cache_dir_content) == 1 @@ -84,12 +78,8 @@ class TestNuphar(unittest.TestCase): nuphar_settings = 'nuphar_cache_path:{}'.format(cache_dir) + ', nuphar_cache_force_no_jit:{}'.format('on') onnxrt.capi._pybind_state.set_nuphar_settings(nuphar_settings) sess = onnxrt.InferenceSession(bidaf_int8_scan_only_model) # JIT cache happens when initializing session - start = timer() - for i in range(jit_repeats): - sess.run([], feed) - jit_time = timer() - start + sess.run([], feed) - self.assertLess(jit_time, non_jit_time) def test_rnn_benchmark(self): # make sure benchmarking scripts works diff --git a/setup.py b/setup.py index aa7cc8cdab..b026797eb9 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,10 @@ elif '--use_ngraph' in sys.argv: elif '--use_openvino' in sys.argv: package_name = 'onnxruntime-openvino' +elif '--use_nuphar' in sys.argv: + package_name = 'onnxruntime-nuphar' + sys.argv.remove('--use_nuphar') + if '--nightly_build' in sys.argv: package_name = 'ort-nightly' nightly_build = True @@ -114,11 +118,15 @@ if platform.system() == 'Linux': libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.so.0', 'libmklml_intel.so', 'libiomp5.so'] # nGraph Libs libs.extend(['libngraph.so', 'libcodegen.so', 'libcpu_backend.so', 'libmkldnn.so', 'libtbb_debug.so', 'libtbb_debug.so.2', 'libtbb.so', 'libtbb.so.2']) + # Nuphar Libs + libs.extend(['libtvm.so']) elif platform.system() == "Darwin": libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.0.dylib'] # TODO add libmklml and libiomp5 later. else: libs = ['onnxruntime_pybind11_state.pyd', 'mkldnn.dll', 'mklml.dll', 'libiomp5md.dll'] libs.extend(['ngraph.dll', 'cpu_backend.dll', 'tbb.dll']) + # Nuphar Libs + libs.extend(['tvm.dll']) if is_manylinux2010: data = [] diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 3acd397b37..04bb3b9274 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -751,7 +751,7 @@ def run_server_model_tests(build_dir, configs): run_subprocess([sys.executable, 'model_zoo_tests.py', server_app_path, test_raw_data_folder, server_test_data_folder, python_package_path, server_test_folder], cwd=server_test_folder, dll_path=None) -def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, use_openvino, nightly_build = False): +def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, use_openvino, use_nuphar, nightly_build = False): for config in configs: cwd = get_config_build_dir(build_dir, config) if is_windows(): @@ -767,6 +767,8 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use args.append('--use_ngraph') elif use_openvino: args.append('--use_openvino') + elif use_nuphar: + args.append('--use_nuphar') run_subprocess(args, cwd=cwd) def build_protoc_for_host(cmake_path, source_dir, build_dir, args): @@ -981,8 +983,8 @@ def main(): mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir) # run nuphar python tests last, as it installs ONNX 1.5.0 - #if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: - # nuphar_run_python_tests(build_dir, configs, args.azure_sas_key) + if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: + nuphar_run_python_tests(build_dir, configs, args.azure_sas_key) if args.build_server: split_server_binary_and_symbol(build_dir, configs) @@ -994,7 +996,7 @@ def main(): if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') - build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_tensorrt, args.use_openvino, nightly_build) + build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_tensorrt, args.use_openvino, args.use_nuphar, nightly_build) if args.gen_doc and (args.build or args.test): generate_documentation(source_dir, build_dir, configs) diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 72ad9ff3dd..53a70e412e 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -6,3 +6,4 @@ jobs: BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_mkldnn --use_tvm --use_automl --build_wheel --enable_language_interop_ops"' DoNugetPack: 'false' ArtifactName: 'drop-linux' + TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 74351e86d4..82efba547b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -5,9 +5,11 @@ parameters: DoNugetPack: 'false' NuPackScript: '' ArtifactName: 'drop-linux' + TimeoutInMinutes: 60 jobs: - job: ${{ parameters.JobName }} + timeoutInMinutes: ${{ parameters.TimeoutInMinutes }} pool: ${{ parameters.AgentPool }} steps: - template: linux-set-variables-and-download.yml diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 601410a8d8..31ea5c0d83 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -46,7 +46,7 @@ apt-get update && apt-get install -y --no-install-recommends \ rsync libunwind8 libpng16-dev libexpat1-dev \ python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ libprotobuf-dev libprotobuf9v5 protobuf-compiler \ - libedit-dev libxml2-dev python3-sympy python3-packaging + libedit-dev libxml2-dev python3-packaging locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 @@ -76,7 +76,8 @@ fi /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0 +/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall sympy==1.1.1 rm -rf /var/lib/apt/lists/* -aria2c -q -d /tmp -o llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-8.0.1/clang+llvm-8.0.1-x86_64-linux-gnu-ubuntu-14.04.tar.xz +aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/6.0.1/clang+llvm-6.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr