update ONNX Runtime Web CI to use same script for package versioning (#9698)

* use ps script for package generating

* allow e2e test has no common pkg

js/web/package.json

@@ -20,7 +20,7 @@
     "pull:wasm": "node ./script/pull-prebuilt-wasm-artifacts",
     "test": "node ./script/prepare-test-data && node ./script/test-runner-cli",
     "test:e2e": "node ./test/e2e/run",
-    "prepack": "node ./script/prepack"
+    "prepack": "node ./script/build && node ./script/prepack"
   },
   "dependencies": {
     "flatbuffers": "^1.12.0",

js/web/test/e2e/run.js

@@ -36,17 +36,21 @@ function getNextUserDataDir() {
 
 const ORT_COMMON_FOLDER = path.resolve(JS_ROOT_FOLDER, 'common');
 const ORT_COMMON_PACKED_FILEPATH_CANDIDATES = globby.sync('onnxruntime-common-*.tgz', { cwd: ORT_COMMON_FOLDER });
-if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length !== 1) {
-  throw new Error('cannot find exactly single package for onnxruntime-common.');
+
+const PACKAGES_TO_INSTALL = [];
+
+if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length === 1) {
+  PACKAGES_TO_INSTALL.push(path.resolve(ORT_COMMON_FOLDER, ORT_COMMON_PACKED_FILEPATH_CANDIDATES[0]));
+} else if (ORT_COMMON_PACKED_FILEPATH_CANDIDATES.length > 1) {
+  throw new Error('multiple packages found for onnxruntime-common.');
 }
-const ORT_COMMON_PACKED_FILEPATH = path.resolve(ORT_COMMON_FOLDER, ORT_COMMON_PACKED_FILEPATH_CANDIDATES[0]);
 
 const ORT_WEB_FOLDER = path.resolve(JS_ROOT_FOLDER, 'web');
 const ORT_WEB_PACKED_FILEPATH_CANDIDATES = globby.sync('onnxruntime-web-*.tgz', { cwd: ORT_WEB_FOLDER });
 if (ORT_WEB_PACKED_FILEPATH_CANDIDATES.length !== 1) {
   throw new Error('cannot find exactly single package for onnxruntime-web.');
 }
-const ORT_WEB_PACKED_FILEPATH = path.resolve(ORT_WEB_FOLDER, ORT_WEB_PACKED_FILEPATH_CANDIDATES[0]);
+PACKAGES_TO_INSTALL.push(path.resolve(ORT_WEB_FOLDER, ORT_WEB_PACKED_FILEPATH_CANDIDATES[0]));
 
 // we start here:
 
@@ -55,7 +59,7 @@ async function main() {
   await runInShell(`npm install"`);
 
   // npm install with "--cache" to install packed packages with an empty cache folder
-  await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" "${ORT_COMMON_PACKED_FILEPATH}" "${ORT_WEB_PACKED_FILEPATH}"`);
+  await runInShell(`npm install --cache "${NPM_CACHE_FOLDER}" ${PACKAGES_TO_INSTALL.map(i => `"${i}"`).join(' ')}`);
 
   // prepare .wasm files for path override testing
   prepareWasmPathOverrideFiles();

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -145,14 +145,14 @@ jobs:
         python3 -m pip install $(Build.SourcesDirectory)/cmake/external/onnx
         sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer
       displayName: 'Build and Test MacOS'
-      
+
     - template: mac-packaging.yml
       parameters :
         AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
         MacosArch: arm64
         BuildJava: true
         BuildNodejs: true
-        
+
     - template: mac-packaging.yml
       parameters :
         AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
@@ -235,7 +235,7 @@ jobs:
       inputs:
         pathtoPublish: '$(Build.BinariesDirectory)/artifacts'
         artifactName: 'onnxruntime-ios-full-xcframework'
-        
+
     - template: component-governance-component-detection-steps.yml
       parameters :
         condition : 'succeeded'
@@ -333,7 +333,7 @@ jobs:
       buildType: 'current'
       artifactName: 'drop-onnxruntime-java-osx-x86_64'
       targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64'
- 
+
   - task: PowerShell@2
     displayName: 'PowerShell Script'
     inputs:
@@ -417,7 +417,7 @@ jobs:
     inputs:
       artifactName: 'onnxruntime-linux-x64'
       targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
-      
+
   - task: DownloadPipelineArtifact@0
     displayName: 'Download Pipeline Artifact - NuGet'
     inputs:

tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml

@@ -1,8 +1,4 @@
 parameters:
-- name: NpmVersionSuffix
-  type: string
-  default: ''
-
 - name: CommitOverride
   type: boolean
   default: false
@@ -11,6 +7,10 @@ parameters:
   type: string
   default: 'Debug'
 
+- name: NpmPackagingMode
+  type: string
+  default: 'dev'
+
 jobs:
 - job: build_onnxruntime_web
   pool: 'Win-CPU-2019'
@@ -100,29 +100,13 @@ jobs:
      node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)"
     workingDirectory: '$(Build.SourcesDirectory)\js\web'
     displayName: 'Check out of dated documents'
-  - task: CopyFiles@2
+  - task: PowerShell@2
     inputs:
-      sourceFolder: $(Pipeline.Workspace)
-      contents: __commit.txt
-      targetFolder: $(Build.SourcesDirectory)\js\common
-    displayName: 'Copy commit ID to js/common'
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Pipeline.Workspace)
-      contents: __commit.txt
-      targetFolder: $(Build.SourcesDirectory)\js\web
-    displayName: 'Copy commit ID to js/web'
-  - script: |
-     set /p __ort_current_version__=<$(Build.SourcesDirectory)\VERSION_NUMBER
-     node -p "a=new Date().toISOString();YYYYMMDD=a.slice(0,4)+a.slice(5,7)+a.slice(8,10);REV=0;`${process.env.__ort_current_version__}${{ parameters.NpmVersionSuffix }}`" >$(Build.SourcesDirectory)\VERSION_NUMBER
-     python .\tools\python\update_version.py
-    workingDirectory: '$(Build.SourcesDirectory)'
-    displayName: 'update package version'
-    condition: ne('${{ parameters.NpmVersionSuffix }}', '')
-  - script: |
-     npm run build
-    workingDirectory: '$(Build.SourcesDirectory)\js\web'
-    displayName: 'Build ort-web'
+      filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\js\pack-npm-packages.ps1'
+      arguments: '$(NpmPackagingMode) $(Build.SourcesDirectory) web'
+      workingDirectory: $(Build.BinariesDirectory)
+      errorActionPreference: stop
+    displayName: 'Pack NPM packages'
   - script: |
      npm test
     workingDirectory: '$(Build.SourcesDirectory)\js\web'
@@ -135,16 +119,6 @@ jobs:
      npm test -- --wasm-enable-proxy -b=wasm
     workingDirectory: '$(Build.SourcesDirectory)\js\web'
     displayName: 'Run ort-web tests - WebAssembly: proxy'
-  - script: |
-      npm pack
-    workingDirectory: '$(Build.SourcesDirectory)\js\common'
-    displayName: 'Generate NPM package (onnxruntime-common)'
-    condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release'))
-  - script: |
-      npm pack
-    workingDirectory: '$(Build.SourcesDirectory)\js\web'
-    displayName: 'Generate NPM package (onnxruntime-web)'
-    condition: and(succeeded(), eq('${{ parameters.BuildConfig }}', 'Release'))
   - script: |
       npm run test:e2e
     workingDirectory: '$(Build.SourcesDirectory)\js\web'

tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml

@@ -15,11 +15,13 @@ variables:
   #   VersionSuffix
 
   ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
-    VersionSuffix: '-dev.${YYYYMMDD}.${REV}'
+    NpmPackagingMode: 'dev'
   ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}:
-    VersionSuffix: '-rc'
+    NpmPackagingMode: 'rc'
   ${{ if eq(parameters.NpmPublish, 'production (@latest)') }}:
-    VersionSuffix: ''
+    NpmPackagingMode: 'release'
+  ${{ if eq(parameters.NpmPublish, 'custom') }}:
+    NpmPackagingMode: '$(VersionSuffix)'
 
 stages:
 - stage: Extract_commit
@@ -58,6 +60,7 @@ stages:
     parameters:
       CommitOverride: true
       BuildConfig: 'Debug'
+      NpmPackagingMode: '$(NpmPackagingMode)'
 
 - stage: Build_wasm_Release
   dependsOn: Extract_commit
@@ -73,9 +76,9 @@ stages:
   jobs:
   - template: templates/win-web-ci.yml
     parameters:
-      NpmVersionSuffix: '$(VersionSuffix)'
       CommitOverride: true
       BuildConfig: 'Release'
+      NpmPackagingMode: '$(NpmPackagingMode)'
 
 - stage: Test_web_BrowserStack
   dependsOn: Build_wasm_Release

tools/ci_build/github/js/pack-npm-packages.ps1

@@ -3,23 +3,54 @@
 
 # This script makes NPM packages for onnxruntime-common and onnxruntime-(node|web)
 #
-# Release Mode:
+# Release Mode (release):
 #   Do not update version number. Version number should be matching $(ORT_ROOT)/VERSION_NUMBER
 #   Always generate packages for onnxruntime-common and onnxruntime-(node|web)
 #
-# Dev Mode:
+# Release Candidate Mode (rc):
+#   Update version number to {VERSION_BASE}-rc.{YYYYMMDD}-{COMMIT}
+#   Always generate packages for onnxruntime-common and onnxruntime-(node|web)
+#
+# Dev Mode (dev):
 #   Compare current content with latest @dev package for onnxruntime-common. If no change, we
 #   don't publish a new version of onnxruntime-common. Instead, we update the dependency version
 #   to use the existing version.
+#   Update version number to {VERSION_BASE}-dev.{YYYYMMDD}-{COMMIT}
+#
+# Custom Mode:
+#   Use first commandline parameter as version number suffix.
+#   Always generate packages for onnxruntime-common and onnxruntime-(node|web)
 #
 
 if ($Args.Count -ne 3) {
     throw "This script requires 3 arguments, but got $($Args.Count)"
 }
-$MODE=$Args[0] # "dev" or "release"
+$MODE=$Args[0] # "dev" or "release" or "rc"; otherwise it is considered as a version number
 $ORT_ROOT=$Args[1] # eg. D:\source\onnxruntime
 $TARGET=$Args[2] # "node" or "web"
 
+Function Generate-Package-Version-Number {
+    pushd $ORT_ROOT
+    $version_base=Get-Content .\VERSION_NUMBER
+    $version_timestamp=git show -s --format=%ct HEAD
+    $version_commit=git rev-parse HEAD
+    $version_commit_short=git rev-parse --short HEAD
+    popd
+    $version_date=[DateTimeOffset]::FromUnixTimeSeconds($version_timestamp).ToString("yyyyMMdd")
+    if ($MODE -eq 'dev') {
+        $version_number="$version_base-dev.$version_date-$version_commit_short"
+    } elseif ($MODE -eq 'rc') {
+        $version_number="$version_base-rc.$version_date-$version_commit_short"
+    } elseif ($MODE -eq 'release')  {
+        $version_number="$version_base"
+    } else {
+        $version_number="$version_base$MODE"
+    }
+
+    Write-Host "Generated version number for '$MODE': $version_number"
+    return @{ version = $version_number; commit = $version_commit }
+}
+
 $JS_COMMON_DIR="$ORT_ROOT\js\common"
 $JS_TARGET_DIR="$ORT_ROOT\js\$TARGET"
 
@@ -48,19 +79,12 @@ if ($MODE -eq "dev") {
     Write-Host "Tarball downloaded"
 
     # generate @dev version
-    pushd $ORT_ROOT
-    $dev_version_base=Get-Content .\VERSION_NUMBER
-    $dev_version_date=$(Get-Date).ToUniversalTime().ToString("yyyyMMdd")
-    $dev_version_commit=git rev-parse HEAD
-    $dev_version_commit_short=git rev-parse --short HEAD
-    popd
-    $dev_version_number="$dev_version_base-dev.$dev_version_date-$dev_version_commit_short"
-    Write-Host "Generated @dev current version: $dev_version_number"
+    $version_number=Generate-Package-Version-Number
 
     # make package for latest
     pushd $JS_COMMON_DIR
     npm version $ort_common_latest_version
-    echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
+    echo $($version_number.commit) | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
     npm pack
     popd
 
@@ -115,7 +139,7 @@ if ($MODE -eq "dev") {
         Write-Host "Need update to onnxruntime-common@dev"
         # need to publish a new version for onnxruntime-common
         pushd $JS_COMMON_DIR
-        npm version $dev_version_number
+        npm version $($version_number.version)
         # file __commit.txt is already generated
         npm pack
         popd
@@ -123,11 +147,11 @@ if ($MODE -eq "dev") {
 
     # make package for target
     pushd $JS_TARGET_DIR
-    npm version $dev_version_number
-    echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
+    npm version $($version_number.version)
+    echo $($version_number.commit) | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
     npm pack
     popd
-} else {
+} elseif ($MODE -eq "release") {
     # release mode. always publish new package
     pushd $JS_COMMON_DIR
     npm pack
@@ -135,4 +159,17 @@ if ($MODE -eq "dev") {
     pushd $JS_TARGET_DIR
     npm pack
     popd
+} else {
+    # release candidate mode or custom mode. always publish new package
+    $version_number=Generate-Package-Version-Number
+
+    pushd $JS_COMMON_DIR
+    npm version $($version_number.version)
+    npm pack
+    popd
+
+    pushd $JS_TARGET_DIR
+    npm version $($version_number.version)
+    npm pack
+    popd
 }