From c571b993363d1f1095c2921c9ff3f9770c02eb85 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 7 Sep 2022 08:33:27 +0800 Subject: [PATCH] Refactor setup_test_data (#12818) * refactory setup_test_data * mv setup test data to test stage * model link for C# test * add comment --- tools/ci_build/build.py | 36 ++++++++++------ .../azure-pipelines/linux-ci-pipeline.yml | 3 +- .../azure-pipelines/templates/win-cpu-ci.yml | 9 +++- .../azure-pipelines/templates/win-gpu-ci.yml | 43 +++++++++++-------- 4 files changed, 54 insertions(+), 37 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 6a42502b14..831bfc097d 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -757,24 +757,30 @@ def install_python_deps(numpy_version=""): run_subprocess([sys.executable, "-m", "pip", "install"] + dep_packages) -def setup_test_data(build_dir, configs): - # create a shortcut for test models if there is a 'models' - # folder in build_dir +def setup_test_data(source_onnx_model_dir, dest_model_dir_name, build_dir, configs): + # create the symlink/shortcut of onnx models dir under build_dir + # currently, there're 2 sources of onnx models, one is build in OS image, another is + # from {source_dir}/js/test, which is downloaded from onnx web. if is_windows(): - src_model_dir = os.path.join(build_dir, "models") - if os.path.exists("C:\\local\\models") and not os.path.exists(src_model_dir): - log.debug("creating shortcut %s -> %s" % ("C:\\local\\models", src_model_dir)) - run_subprocess(["mklink", "/D", "/J", src_model_dir, "C:\\local\\models"], shell=True) + src_model_dir = os.path.join(build_dir, dest_model_dir_name) + if os.path.exists(source_onnx_model_dir) and not os.path.exists(src_model_dir): + log.debug("creating shortcut %s -> %s" % (source_onnx_model_dir, src_model_dir)) + run_subprocess(["mklink", "/D", "/J", src_model_dir, source_onnx_model_dir], shell=True) for config in configs: config_build_dir = get_config_build_dir(build_dir, config) os.makedirs(config_build_dir, exist_ok=True) - dest_model_dir = os.path.join(config_build_dir, "models") - if os.path.exists("C:\\local\\models") and not os.path.exists(dest_model_dir): - log.debug("creating shortcut %s -> %s" % ("C:\\local\\models", dest_model_dir)) - run_subprocess(["mklink", "/D", "/J", dest_model_dir, "C:\\local\\models"], shell=True) + dest_model_dir = os.path.join(config_build_dir, dest_model_dir_name) + if os.path.exists(source_onnx_model_dir) and not os.path.exists(dest_model_dir): + log.debug("creating shortcut %s -> %s" % (source_onnx_model_dir, dest_model_dir)) + run_subprocess(["mklink", "/D", "/J", dest_model_dir, source_onnx_model_dir], shell=True) elif os.path.exists(src_model_dir) and not os.path.exists(dest_model_dir): log.debug("creating shortcut %s -> %s" % (src_model_dir, dest_model_dir)) run_subprocess(["mklink", "/D", "/J", dest_model_dir, src_model_dir], shell=True) + else: + src_model_dir = os.path.join(build_dir, dest_model_dir_name) + if os.path.exists(source_onnx_model_dir) and not os.path.exists(src_model_dir): + log.debug(f"create symlink {source_onnx_model_dir} -> {src_model_dir}") + os.symlink(source_onnx_model_dir, src_model_dir, target_is_directory=True) def use_dev_mode(args): @@ -2584,9 +2590,6 @@ def main(): if args.enable_pybind and is_windows(): install_python_deps(args.numpy_version) - if args.enable_onnx_tests: - setup_test_data(build_dir, configs) - if args.use_cuda and args.cuda_version is None: if is_windows(): # cuda_version is used while generating version_info.py on Windows. @@ -2700,6 +2703,11 @@ def main(): build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, args.target) if args.test: + if args.enable_onnx_tests: + source_onnx_model_dir = "C:\\local\\models" if is_windows() else "/data/models" + dest_model_dir_name = "models" + setup_test_data(source_onnx_model_dir, dest_model_dir_name, build_dir, configs) + run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs) if args.enable_pybind and not args.skip_onnx_tests and args.use_nuphar: diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 6db5e1bbc2..ffe9885661 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -5,7 +5,7 @@ resources: endpoint: Microsoft name: pypa/manylinux ref: a8099af1b3e25f0489717ad9c4f9a2e25a8c5b36 - + jobs: - job: Linux_Build timeoutInMinutes: 120 @@ -74,7 +74,6 @@ jobs: mkdir $(Build.BinariesDirectory)/requirements_torch_cpu/ cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt - ln -sf /data/models $(Build.BinariesDirectory) cd $(Build.SourcesDirectory)/java /usr/local/gradle/bin/gradle "cmakeCheck" "-DcmakeBuildDir=$(Build.BinariesDirectory)/Release" diff --git a/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml index 225ddfae2a..d77d00dd6b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml @@ -161,6 +161,11 @@ jobs: msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)' workingDirectory: '$(Build.SourcesDirectory)\csharp' + # C# test isn't launched by build.py, so models link has to be added. + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' - ${{ if and(eq(parameters.BuildConfig, 'RelWithDebInfo'), eq(parameters.RunOnnxRuntimeTests, true)) }}: - task: DotNetCoreCLI@2 @@ -198,8 +203,8 @@ jobs: **/*.pdb **/*.dll - #Manually set msBuildCommandline so that we can also set CAExcludePath - #build_dir must be a sub folder of $(Build.SourcesDirectory) + # Manually set msBuildCommandline so that we can also set CAExcludePath + # build_dir must be a sub folder of $(Build.SourcesDirectory) - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-gpu-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-gpu-ci.yml index 5a8415bda8..9d04c0eee8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-gpu-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-gpu-ci.yml @@ -20,7 +20,7 @@ parameters: - name: isX86 type: boolean default: false - + - name: isTraining type: boolean default: false @@ -37,11 +37,11 @@ parameters: - name: RunStaticCodeAnalysis displayName: Run Static Code Analysis type: boolean - default: true - + default: true + - name: ORT_EP_NAME type: string - + - name: MachinePool type: string @@ -55,7 +55,7 @@ jobs: variables: OrtPackageId: 'Microsoft.ML.OnnxRuntime' OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' DocUpdateNeeded: ${{ parameters.DocUpdateNeeded }} @@ -63,7 +63,7 @@ jobs: workspace: clean: all pool: ${{ parameters.MachinePool }} - timeoutInMinutes: 300 + timeoutInMinutes: 300 steps: - task: UsePythonVersion@0 inputs: @@ -94,10 +94,10 @@ jobs: - script: | set ORT_DOXY_SRC=$(Build.SourcesDirectory) set ORT_DOXY_OUT=$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }} - mkdir %ORT_DOXY_SRC% + mkdir %ORT_DOXY_SRC% mkdir %ORT_DOXY_OUT% "C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg - + workingDirectory: '$(Build.SourcesDirectory)' displayName: 'API Documentation Check and generate' @@ -176,7 +176,7 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\csharp' - task: MSBuild@1 - displayName: 'Build C#' + displayName: 'Build C#' inputs: solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' configuration: '${{ parameters.BuildConfig }}' @@ -184,6 +184,11 @@ jobs: msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' + # C# test isn't launched by build.py, so models link has to be added. + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' - ${{ if and(eq(parameters.BuildConfig, 'RelWithDebInfo'), eq(parameters.RunOnnxRuntimeTests, true)) }}: - task: DotNetCoreCLI@2 @@ -191,7 +196,7 @@ jobs: inputs: command: test projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj' - configuration: '${{ parameters.BuildConfig }}' + configuration: '${{ parameters.BuildConfig }}' arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_${{ parameters.ORT_EP_NAME }} --blame' workingDirectory: '$(Build.SourcesDirectory)\csharp' @@ -199,14 +204,14 @@ jobs: - powershell: | python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml -qq Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - + workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' displayName: 'Install onnxruntime wheel' - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}: - powershell: | python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} - + workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' displayName: 'Run tests' @@ -222,9 +227,9 @@ jobs: **/*.dll - #Manually set msBuildCommandline so that we can also set CAExcludePath - #build_dir must be a sub folder of $(Build.SourcesDirectory) - #TODO: move this step to a CPU-only machine to save GPU resources. + # Manually set msBuildCommandline so that we can also set CAExcludePath + # build_dir must be a sub folder of $(Build.SourcesDirectory) + # TODO: move this step to a CPU-only machine to save GPU resources. - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' inputs: @@ -235,16 +240,16 @@ jobs: rulesetName: Custom customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset publishXML: true - + - task: SdtReport@2 displayName: 'Create Security Analysis Report' inputs: SDLNativeRules: true - + - task: PublishSecurityAnalysisLogs@3 displayName: 'Publish Security Analysis Logs' continueOnError: true - + - task: PostAnalysis@2 displayName: 'Guardian Break v2' inputs: @@ -275,4 +280,4 @@ jobs: condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) inputs: pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md' - artifactName: 'ContribOperators.md' \ No newline at end of file + artifactName: 'ContribOperators.md'