Use cuda9 1 in c api packaging (#571)

* use CUDA 9.1 for both linux and windows

* added powershell scripts for cuda props setup/cleanup

* fix yml syntax

* set path to cuda9.1 bin

* correct label

* ad --cuda_version

* added some log to browse the directory

* disabled jobs other than win gpu to save some resource while testing

* add msvc_toolset=14.11

* added more logs

* log the props file

* remove setting vcvarsall

* try some modificationi n build.py

* fix typo

* let the config Step modify envoronment

* set some more env vars manually

* try reordering vcvars after cuda props copying

* use single script for build and test

* single line script

* remove extra quote

* cleanup trial changes

tools/ci_build/build.py

@@ -79,9 +79,6 @@ Use the individual flags to only run the specified stages.
     # Python bindings
     parser.add_argument("--enable_pybind", action='store_true', help="Enable Python Bindings.")
     parser.add_argument("--build_wheel", action='store_true', help="Build Python Wheel. ")
-    parser.add_argument("--numpy_version", default='1.15.0', help="Installs a specific version of numpy "
-                        "before building the python binding.")
-    parser.add_argument("--skip-keras-test", action='store_true', help="Skip tests with Keras if keras is installed")
 
     # C-Sharp bindings
     parser.add_argument("--build_csharp", action='store_true', help="Build C#.Net DLL and NuGet package")
@@ -198,9 +195,8 @@ def install_ubuntu_deps(args):
         except Exception as e:
             raise BuildError("Error setting up required APT packages. {}".format(str(e)))
 
-def install_python_deps(numpy_version=""):
-    dep_packages = ['setuptools', 'wheel']
-    dep_packages.append('numpy==%s' % numpy_version if numpy_version else 'numpy')
+def install_python_deps():
+    dep_packages = ['setuptools', 'wheel', 'numpy==1.15.0']
     run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host', 'files.pythonhosted.org'] + dep_packages)
 
 def check_md5(filename, expected_md5):
@@ -469,16 +465,15 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab
                 run_subprocess([os.path.join(cwd,'onnx_test_runner'), 'test_models'], cwd=cwd)
                 if config != 'Debug':
                     run_subprocess([sys.executable, 'onnx_backend_test_series.py'], cwd=cwd, dll_path=dll_path)
-            if not args.skip_keras_test:
-                try:
-                    import onnxmltools
-                    import keras
-                    onnxml_test = True
-                except ImportError:
-                    warnings.warn("onnxmltools and keras are not installed. Following test cannot be run.")
-                    onnxml_test = False
-                if onnxml_test:
-                    run_subprocess([sys.executable, 'onnxruntime_test_python_keras.py'], cwd=cwd, dll_path=dll_path)
+            try:
+                import onnxmltools
+                import keras
+                onnxml_test = True
+            except ImportError:
+                warnings.warn("onnxmltools and keras are not installed. Following test cannot be run.")
+                onnxml_test = False
+            if onnxml_test:
+                run_subprocess([sys.executable, 'onnxruntime_test_python_keras.py'], cwd=cwd, dll_path=dll_path)
 
 def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_parallel_executor_test, num_parallel_models):
     for config in configs:
@@ -575,7 +570,7 @@ def main():
             if not is_docker():
                 install_python_deps()
         if (args.enable_pybind and is_windows()):
-            install_python_deps(args.numpy_version)
+            install_python_deps()
         if (not args.skip_submodule_sync):
             update_submodules(source_dir)
 

tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml

@@ -30,11 +30,11 @@ jobs:
 
 
 - job: Linux_C_API_Packaging_GPU_x64
-  pool: Linux-GPU-CUDA10
+  pool: Linux-GPU
   steps:
     - template: templates/set-test-data-variables-step.yml
 
-    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x " --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"'
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -x " --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)"'
       displayName: 'Build and Test Linux on Docker'
     - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
       parameters:
@@ -126,7 +126,7 @@ jobs:
 
 
 - job: Windows_Packaging_GPU_x64
-  pool: 'Win-GPU-CUDA10'
+  pool: 'Win-GPU'
   variables:
     buildDirectory: '$(Build.BinariesDirectory)'
     buildConfig: 'RelWithDebInfo'
@@ -135,21 +135,45 @@ jobs:
   steps:
     - template: templates/set-test-data-variables-step.yml
 
+    - task: CmdLine@2
+      displayName: 'Set CUDA 9.1 path'
+      inputs:
+        script: |
+          set PATH=C:\local\cuda-9.1.85-windows10-x64-0\bin;C:\local\cudnn-9.1-windows10-x64-v7.1\cuda\bin;%PATH%
+        modifyEnvironment: true
+        workingDirectory: '$(Build.BinariesDirectory)'
+
+    - task: PowerShell@2
+      displayName: 'Set CUDA 9.1 MSBuild properties'
+      inputs:
+        targetType: 'filePath'
+        filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/set_cuda_path.ps1'
+        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.1.85-windows10-x64-0 -CudaVersion 9.1'
+
     - template: templates/windows-build-tools-setup-steps.yml
       parameters:
-        EnvSetupScript: 'setup_env_cuda.bat'
+        EnvSetupScript: 'setup_env.bat'
         buildArch: 'amd64'  # amd64 is needed for vcvars target arch
-        setVcvars: true 
+        setVcvars: true
 
-    - template: templates/windows-build-and-test-steps.yml
-      parameters:
-        buildAdditionalParams: ' --use_openmp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" '
-        buildArch: $(buildArch)
-        buildConfig: $(buildConfig)
+    - task: CmdLine@2
+      displayName: 'Build and Test OnnxRuntime'
+      inputs:
+        script: |
+          $(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --use_openmp --msvc_toolset=14.11 --use_cuda --cuda_version 9.1 --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda" 
+
+        workingDirectory: '$(Build.BinariesDirectory)'
 
     - template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
       parameters:
         buildConfig: $(buildConfig)
         artifactName: 'onnxruntime-win-gpu-$(buildArch)'
 
+    - task: PowerShell@2
+      displayName: 'Clean up Cuda Path 9.1'
+      inputs:
+        targetType: 'filePath'
+        filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1'
+        arguments: '-CudaVersion 9.1'
+
     - template: templates/clean-agent-build-directory-step.yml