From 31dc768e0760cb76114f1cf9e12524b4940ca0ad Mon Sep 17 00:00:00 2001 From: Yulong Wang <7679871+fs-eire@users.noreply.github.com> Date: Wed, 10 Nov 2021 12:52:34 -0800 Subject: [PATCH] update ONNX Runtime Web CI to use same script for package versioning (#9698) * use ps script for package generating * allow e2e test has no common pkg --- js/web/package.json | 2 +- js/web/test/e2e/run.js | 14 ++-- .../azure-pipelines/templates/c-api-cpu.yml | 10 +-- .../azure-pipelines/templates/win-web-ci.yml | 46 +++---------- .../azure-pipelines/web-ci-pipeline.yml | 11 +-- .../ci_build/github/js/pack-npm-packages.ps1 | 69 ++++++++++++++----- 6 files changed, 85 insertions(+), 67 deletions(-) diff --git a/js/web/package.json b/js/web/package.json index 6f9332421e..da4eecebdb 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -20,7 +20,7 @@ "pull:wasm": "node ./script/pull-prebuilt-wasm-artifacts", "test": "node ./script/prepare-test-data && node ./script/test-runner-cli", "test:e2e": "node ./test/e2e/run", - "prepack": "node ./script/prepack" + "prepack": "node ./script/build && node ./script/prepack" }, "dependencies": { "flatbuffers": "^1.12.0", diff --git a/js/web/test/e2e/run.js b/js/web/test/e2e/run.js index bafd7c04e6..5b2544a1af 100644 --- a/js/web/test/e2e/run.js +++ b/js/web/test/e2e/run.js @@ -36,17 +36,21 @@ function getNextUserDataDir() { const ORT_COMMON_FOLDER = path.resolve(JS_ROOT_FOLDER, 'common'); const ORT_COMMON_PACKED_FILEPATH_CANDIDATES = globby.sync('onnxruntime-common-*.tgz', { cwd: ORT_COMMON_FOLDER }); -if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length !== 1) { - throw new Error('cannot find exactly single package for onnxruntime-common.'); + +const PACKAGES_TO_INSTALL = []; + +if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length === 1) { + PACKAGES_TO_INSTALL.push(path.resolve(ORT_COMMON_FOLDER, ORT_COMMON_PACKED_FILEPATH_CANDIDATES[0])); +} else if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length > 1) { + throw new Error('multiple packages found for onnxruntime-common.'); } -const ORT_COMMON_PACKED_FILEPATH = path.resolve(ORT_COMMON_FOLDER, ORT_COMMON_PACKED_FILEPATH_CANDIDATES[0]); const ORT_WEB_FOLDER = path.resolve(JS_ROOT_FOLDER, 'web'); const ORT_WEB_PACKED_FILEPATH_CANDIDATES = globby.sync('onnxruntime-web-*.tgz', { cwd: ORT_WEB_FOLDER }); if (ORT_WEB_PACKED_FILEPATH_CANDIDATES.length !== 1) { throw new Error('cannot find exactly single package for onnxruntime-web.'); } -const ORT_WEB_PACKED_FILEPATH = path.resolve(ORT_WEB_FOLDER, ORT_WEB_PACKED_FILEPATH_CANDIDATES[0]); +PACKAGES_TO_INSTALL.push(path.resolve(ORT_WEB_FOLDER, ORT_WEB_PACKED_FILEPATH_CANDIDATES[0])); // we start here: @@ -55,7 +59,7 @@ async function main() { await runInShell(`npm install"`); // npm install with "--cache" to install packed packages with an empty cache folder - await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" "${ORT_COMMON_PACKED_FILEPATH}" "${ORT_WEB_PACKED_FILEPATH}"`); + await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" ${PACKAGES_TO_INSTALL.map(i => `"${i}"`).join(' ')}`); // prepare .wasm files for path override testing prepareWasmPathOverrideFiles(); diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 80ce8a6c42..03b5dbac76 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -145,14 +145,14 @@ jobs: python3 -m pip install $(Build.SourcesDirectory)/cmake/external/onnx sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer displayName: 'Build and Test MacOS' - + - template: mac-packaging.yml parameters : AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64 MacosArch: arm64 BuildJava: true BuildNodejs: true - + - template: mac-packaging.yml parameters : AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" @@ -235,7 +235,7 @@ jobs: inputs: pathtoPublish: '$(Build.BinariesDirectory)/artifacts' artifactName: 'onnxruntime-ios-full-xcframework' - + - template: component-governance-component-detection-steps.yml parameters : condition : 'succeeded' @@ -333,7 +333,7 @@ jobs: buildType: 'current' artifactName: 'drop-onnxruntime-java-osx-x86_64' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64' - + - task: PowerShell@2 displayName: 'PowerShell Script' inputs: @@ -417,7 +417,7 @@ jobs: inputs: artifactName: 'onnxruntime-linux-x64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml index 58cb14991f..b7691af037 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml @@ -1,8 +1,4 @@ parameters: -- name: NpmVersionSuffix - type: string - default: '' - - name: CommitOverride type: boolean default: false @@ -11,6 +7,10 @@ parameters: type: string default: 'Debug' +- name: NpmPackagingMode + type: string + default: 'dev' + jobs: - job: build_onnxruntime_web pool: 'Win-CPU-2019' @@ -100,29 +100,13 @@ jobs: node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Check out of dated documents' - - task: CopyFiles@2 + - task: PowerShell@2 inputs: - sourceFolder: $(Pipeline.Workspace) - contents: __commit.txt - targetFolder: $(Build.SourcesDirectory)\js\common - displayName: 'Copy commit ID to js/common' - - task: CopyFiles@2 - inputs: - sourceFolder: $(Pipeline.Workspace) - contents: __commit.txt - targetFolder: $(Build.SourcesDirectory)\js\web - displayName: 'Copy commit ID to js/web' - - script: | - set /p __ort_current_version__=<$(Build.SourcesDirectory)\VERSION_NUMBER - node -p "a=new Date().toISOString();YYYYMMDD=a.slice(0,4)+a.slice(5,7)+a.slice(8,10);REV=0;`${process.env.__ort_current_version__}${{ parameters.NpmVersionSuffix }}`" >$(Build.SourcesDirectory)\VERSION_NUMBER - python .\tools\python\update_version.py - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'update package version' - condition: ne('${{ parameters.NpmVersionSuffix }}', '') - - script: | - npm run build - workingDirectory: '$(Build.SourcesDirectory)\js\web' - displayName: 'Build ort-web' + filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\js\pack-npm-packages.ps1' + arguments: '$(NpmPackagingMode) $(Build.SourcesDirectory) web' + workingDirectory: $(Build.BinariesDirectory) + errorActionPreference: stop + displayName: 'Pack NPM packages' - script: | npm test workingDirectory: '$(Build.SourcesDirectory)\js\web' @@ -135,16 +119,6 @@ jobs: npm test -- --wasm-enable-proxy -b=wasm workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Run ort-web tests - WebAssembly: proxy' - - script: | - npm pack - workingDirectory: '$(Build.SourcesDirectory)\js\common' - displayName: 'Generate NPM package (onnxruntime-common)' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - - script: | - npm pack - workingDirectory: '$(Build.SourcesDirectory)\js\web' - displayName: 'Generate NPM package (onnxruntime-web)' - condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release')) - script: | npm run test:e2e workingDirectory: '$(Build.SourcesDirectory)\js\web' diff --git a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml index 774eaba256..52fec876bb 100644 --- a/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml @@ -15,11 +15,13 @@ variables: # VersionSuffix ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}: - VersionSuffix: '-dev.${YYYYMMDD}.${REV}' + NpmPackagingMode: 'dev' ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}: - VersionSuffix: '-rc' + NpmPackagingMode: 'rc' ${{ if eq(parameters.NpmPublish, 'production (@latest)') }}: - VersionSuffix: '' + NpmPackagingMode: 'release' + ${{ if eq(parameters.NpmPublish, 'custom') }}: + NpmPackagingMode: '$(VersionSuffix)' stages: - stage: Extract_commit @@ -58,6 +60,7 @@ stages: parameters: CommitOverride: true BuildConfig: 'Debug' + NpmPackagingMode: '$(NpmPackagingMode)' - stage: Build_wasm_Release dependsOn: Extract_commit @@ -73,9 +76,9 @@ stages: jobs: - template: templates/win-web-ci.yml parameters: - NpmVersionSuffix: '$(VersionSuffix)' CommitOverride: true BuildConfig: 'Release' + NpmPackagingMode: '$(NpmPackagingMode)' - stage: Test_web_BrowserStack dependsOn: Build_wasm_Release diff --git a/tools/ci_build/github/js/pack-npm-packages.ps1 b/tools/ci_build/github/js/pack-npm-packages.ps1 index ea86551580..ca1dd96bda 100644 --- a/tools/ci_build/github/js/pack-npm-packages.ps1 +++ b/tools/ci_build/github/js/pack-npm-packages.ps1 @@ -3,23 +3,54 @@ # This script makes NPM packages for onnxruntime-common and onnxruntime-(node|web) # -# Release Mode: +# Release Mode (release): # Do not update version number. Version number should be matching $(ORT_ROOT)/VERSION_NUMBER # Always generate packages for onnxruntime-common and onnxruntime-(node|web) # -# Dev Mode: +# Release Candidate Mode (rc): +# Update version number to {VERSION_BASE}-rc.{YYYYMMDD}-{COMMIT} +# Always generate packages for onnxruntime-common and onnxruntime-(node|web) +# +# Dev Mode (dev): # Compare current content with latest @dev package for onnxruntime-common. If no change, we # don't publish a new version of onnxruntime-common. Instead, we update the dependency version # to use the existing version. +# Update version number to {VERSION_BASE}-dev.{YYYYMMDD}-{COMMIT} +# +# Custom Mode: +# Use first commandline parameter as version number suffix. +# Always generate packages for onnxruntime-common and onnxruntime-(node|web) # if ($Args.Count -ne 3) { throw "This script requires 3 arguments, but got $($Args.Count)" } -$MODE=$Args[0] # "dev" or "release" +$MODE=$Args[0] # "dev" or "release" or "rc"; otherwise it is considered as a version number $ORT_ROOT=$Args[1] # eg. D:\source\onnxruntime $TARGET=$Args[2] # "node" or "web" +Function Generate-Package-Version-Number { + pushd $ORT_ROOT + $version_base=Get-Content .\VERSION_NUMBER + $version_timestamp=git show -s --format=%ct HEAD + $version_commit=git rev-parse HEAD + $version_commit_short=git rev-parse --short HEAD + popd + $version_date=[DateTimeOffset]::FromUnixTimeSeconds($version_timestamp).ToString("yyyyMMdd") + if ($MODE -eq 'dev') { + $version_number="$version_base-dev.$version_date-$version_commit_short" + } elseif ($MODE -eq 'rc') { + $version_number="$version_base-rc.$version_date-$version_commit_short" + } elseif ($MODE -eq 'release') { + $version_number="$version_base" + } else { + $version_number="$version_base$MODE" + } + + Write-Host "Generated version number for '$MODE': $version_number" + return @{ version = $version_number; commit = $version_commit } +} + $JS_COMMON_DIR="$ORT_ROOT\js\common" $JS_TARGET_DIR="$ORT_ROOT\js\$TARGET" @@ -48,19 +79,12 @@ if ($MODE -eq "dev") { Write-Host "Tarball downloaded" # generate @dev version - pushd $ORT_ROOT - $dev_version_base=Get-Content .\VERSION_NUMBER - $dev_version_date=$(Get-Date).ToUniversalTime().ToString("yyyyMMdd") - $dev_version_commit=git rev-parse HEAD - $dev_version_commit_short=git rev-parse --short HEAD - popd - $dev_version_number="$dev_version_base-dev.$dev_version_date-$dev_version_commit_short" - Write-Host "Generated @dev current version: $dev_version_number" + $version_number=Generate-Package-Version-Number # make package for latest pushd $JS_COMMON_DIR npm version $ort_common_latest_version - echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt + echo $($version_number.commit) | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt npm pack popd @@ -115,7 +139,7 @@ if ($MODE -eq "dev") { Write-Host "Need update to onnxruntime-common@dev" # need to publish a new version for onnxruntime-common pushd $JS_COMMON_DIR - npm version $dev_version_number + npm version $($version_number.version) # file __commit.txt is already generated npm pack popd @@ -123,11 +147,11 @@ if ($MODE -eq "dev") { # make package for target pushd $JS_TARGET_DIR - npm version $dev_version_number - echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt + npm version $($version_number.version) + echo $($version_number.commit) | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt npm pack popd -} else { +} elseif ($MODE -eq "release") { # release mode. always publish new package pushd $JS_COMMON_DIR npm pack @@ -135,4 +159,17 @@ if ($MODE -eq "dev") { pushd $JS_TARGET_DIR npm pack popd +} else { + # release candidate mode or custom mode. always publish new package + $version_number=Generate-Package-Version-Number + + pushd $JS_COMMON_DIR + npm version $($version_number.version) + npm pack + popd + + pushd $JS_TARGET_DIR + npm version $($version_number.version) + npm pack + popd }