Refactor setup_test_data (#12818)

* refactory setup_test_data

* mv setup test data to test stage

* model link for C# test

* add comment

tools/ci_build/build.py

@@ -757,24 +757,30 @@ def install_python_deps(numpy_version=""):
     run_subprocess([sys.executable, "-m", "pip", "install"] + dep_packages)
 
 
-def setup_test_data(build_dir, configs):
-    # create a shortcut for test models if there is a 'models'
-    # folder in build_dir
+def setup_test_data(source_onnx_model_dir, dest_model_dir_name, build_dir, configs):
+    # create the symlink/shortcut of onnx models dir under build_dir
+    # currently, there're 2 sources of onnx models, one is build in OS image, another is
+    # from {source_dir}/js/test, which is downloaded from onnx web.
     if is_windows():
-        src_model_dir = os.path.join(build_dir, "models")
-        if os.path.exists("C:\\local\\models") and not os.path.exists(src_model_dir):
-            log.debug("creating shortcut %s -> %s" % ("C:\\local\\models", src_model_dir))
-            run_subprocess(["mklink", "/D", "/J", src_model_dir, "C:\\local\\models"], shell=True)
+        src_model_dir = os.path.join(build_dir, dest_model_dir_name)
+        if os.path.exists(source_onnx_model_dir) and not os.path.exists(src_model_dir):
+            log.debug("creating shortcut %s -> %s" % (source_onnx_model_dir, src_model_dir))
+            run_subprocess(["mklink", "/D", "/J", src_model_dir, source_onnx_model_dir], shell=True)
         for config in configs:
             config_build_dir = get_config_build_dir(build_dir, config)
             os.makedirs(config_build_dir, exist_ok=True)
-            dest_model_dir = os.path.join(config_build_dir, "models")
-            if os.path.exists("C:\\local\\models") and not os.path.exists(dest_model_dir):
-                log.debug("creating shortcut %s -> %s" % ("C:\\local\\models", dest_model_dir))
-                run_subprocess(["mklink", "/D", "/J", dest_model_dir, "C:\\local\\models"], shell=True)
+            dest_model_dir = os.path.join(config_build_dir, dest_model_dir_name)
+            if os.path.exists(source_onnx_model_dir) and not os.path.exists(dest_model_dir):
+                log.debug("creating shortcut %s -> %s" % (source_onnx_model_dir, dest_model_dir))
+                run_subprocess(["mklink", "/D", "/J", dest_model_dir, source_onnx_model_dir], shell=True)
             elif os.path.exists(src_model_dir) and not os.path.exists(dest_model_dir):
                 log.debug("creating shortcut %s -> %s" % (src_model_dir, dest_model_dir))
                 run_subprocess(["mklink", "/D", "/J", dest_model_dir, src_model_dir], shell=True)
+    else:
+        src_model_dir = os.path.join(build_dir, dest_model_dir_name)
+        if os.path.exists(source_onnx_model_dir) and not os.path.exists(src_model_dir):
+            log.debug(f"create symlink {source_onnx_model_dir} -> {src_model_dir}")
+            os.symlink(source_onnx_model_dir, src_model_dir, target_is_directory=True)
 
 
 def use_dev_mode(args):
@@ -2584,9 +2590,6 @@ def main():
         if args.enable_pybind and is_windows():
             install_python_deps(args.numpy_version)
 
-        if args.enable_onnx_tests:
-            setup_test_data(build_dir, configs)
-
         if args.use_cuda and args.cuda_version is None:
             if is_windows():
                 # cuda_version is used while generating version_info.py on Windows.
@@ -2700,6 +2703,11 @@ def main():
         build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, args.target)
 
     if args.test:
+        if args.enable_onnx_tests:
+            source_onnx_model_dir = "C:\\local\\models" if is_windows() else "/data/models"
+            dest_model_dir_name = "models"
+            setup_test_data(source_onnx_model_dir, dest_model_dir_name, build_dir, configs)
+
         run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs)
 
         if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar:

tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml

@@ -5,7 +5,7 @@ resources:
     endpoint: Microsoft
     name: pypa/manylinux
     ref: a8099af1b3e25f0489717ad9c4f9a2e25a8c5b36
-    
+
 jobs:
 - job: Linux_Build
   timeoutInMinutes: 120
@@ -74,7 +74,6 @@ jobs:
          mkdir $(Build.BinariesDirectory)/requirements_torch_cpu/
          cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
          python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt
-         ln -sf /data/models $(Build.BinariesDirectory)
          cd $(Build.SourcesDirectory)/java
          /usr/local/gradle/bin/gradle "cmakeCheck" "-DcmakeBuildDir=$(Build.BinariesDirectory)/Release"
 

tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml

@@ -161,6 +161,11 @@ jobs:
       msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
+  # C# test isn't launched by build.py, so models link has to be added.
+  - script: |
+     mklink /D /J models C:\local\models
+    workingDirectory: '$(Build.BinariesDirectory)'
+    displayName: 'Create models link'
 
   - ${{ if and(eq(parameters.BuildConfig, 'RelWithDebInfo'), eq(parameters.RunOnnxRuntimeTests, true)) }}:
       - task: DotNetCoreCLI@2
@@ -198,8 +203,8 @@ jobs:
            **/*.pdb
            **/*.dll
 
-      #Manually set msBuildCommandline so that we can also set CAExcludePath
-      #build_dir must be a sub folder of $(Build.SourcesDirectory)
+      # Manually set msBuildCommandline so that we can also set CAExcludePath
+      # build_dir must be a sub folder of $(Build.SourcesDirectory)
       - task: SDLNativeRules@3
         displayName: 'Run the PREfast SDL Native Rules for MSBuild'
         inputs:

tools/ci_build/github/azure-pipelines/templates/win-gpu-ci.yml

@@ -20,7 +20,7 @@ parameters:
 - name: isX86
   type: boolean
   default: false
-  
+
 - name: isTraining
   type: boolean
   default: false
@@ -37,11 +37,11 @@ parameters:
 - name: RunStaticCodeAnalysis
   displayName: Run Static Code Analysis
   type: boolean
-  default: true  
-  
+  default: true
+
 - name: ORT_EP_NAME
   type: string
-  
+
 - name: MachinePool
   type: string
 
@@ -55,7 +55,7 @@ jobs:
   variables:
     OrtPackageId: 'Microsoft.ML.OnnxRuntime'
     OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
-    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true    
+    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     setVcvars: true
     ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
     DocUpdateNeeded: ${{ parameters.DocUpdateNeeded }}
@@ -63,7 +63,7 @@ jobs:
   workspace:
     clean: all
   pool: ${{ parameters.MachinePool }}
-  timeoutInMinutes:  300  
+  timeoutInMinutes:  300
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -94,10 +94,10 @@ jobs:
   - script: |
      set ORT_DOXY_SRC=$(Build.SourcesDirectory)
      set ORT_DOXY_OUT=$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}
-     mkdir %ORT_DOXY_SRC% 
+     mkdir %ORT_DOXY_SRC%
      mkdir %ORT_DOXY_OUT%
      "C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg
-     
+
     workingDirectory: '$(Build.SourcesDirectory)'
     displayName: 'API Documentation Check and generate'
 
@@ -176,7 +176,7 @@ jobs:
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
   - task: MSBuild@1
-    displayName: 'Build C#'    
+    displayName: 'Build C#'
     inputs:
       solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
       configuration: '${{ parameters.BuildConfig }}'
@@ -184,6 +184,11 @@ jobs:
       msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }}'
       workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
+  # C# test isn't launched by build.py, so models link has to be added.
+  - script: |
+     mklink /D /J models C:\local\models
+    workingDirectory: '$(Build.BinariesDirectory)'
+    displayName: 'Create models link'
 
   - ${{ if and(eq(parameters.BuildConfig, 'RelWithDebInfo'), eq(parameters.RunOnnxRuntimeTests, true)) }}:
       - task: DotNetCoreCLI@2
@@ -191,7 +196,7 @@ jobs:
         inputs:
           command: test
           projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj'
-          configuration: '${{ parameters.BuildConfig }}'          
+          configuration: '${{ parameters.BuildConfig }}'
           arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }} --blame'
           workingDirectory: '$(Build.SourcesDirectory)\csharp'
 
@@ -199,14 +204,14 @@ jobs:
       - powershell: |
          python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml -qq
          Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
-       
+
         workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
         displayName: 'Install onnxruntime wheel'
 
   - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}:
       - powershell: |
          python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019"  --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}
-       
+
         workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
         displayName: 'Run tests'
 
@@ -222,9 +227,9 @@ jobs:
            **/*.dll
 
 
-      #Manually set msBuildCommandline so that we can also set CAExcludePath
-      #build_dir must be a sub folder of $(Build.SourcesDirectory)
-      #TODO: move this step to a CPU-only machine to save GPU resources.
+      # Manually set msBuildCommandline so that we can also set CAExcludePath
+      # build_dir must be a sub folder of $(Build.SourcesDirectory)
+      # TODO: move this step to a CPU-only machine to save GPU resources.
       - task: SDLNativeRules@3
         displayName: 'Run the PREfast SDL Native Rules for MSBuild'
         inputs:
@@ -235,16 +240,16 @@ jobs:
           rulesetName: Custom
           customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset
           publishXML: true
-          
+
       - task: SdtReport@2
         displayName: 'Create Security Analysis Report'
         inputs:
           SDLNativeRules: true
-    
+
       - task: PublishSecurityAnalysisLogs@3
         displayName: 'Publish Security Analysis Logs'
         continueOnError: true
-   
+
       - task: PostAnalysis@2
         displayName: 'Guardian Break v2'
         inputs:
@@ -275,4 +280,4 @@ jobs:
     condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true'))
     inputs:
       pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md'
-      artifactName: 'ContribOperators.md'
+      artifactName: 'ContribOperators.md'