[js] release pipeline for web and react native (#10656)

* skip browserstack test at release pipeline

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* pool name as a parameter to run at lotus

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* Update web-ci-pipeline.yml for Azure Pipelines

* create a packaging pipeline for web

* Update web-packaging-pipeline.yml for Azure Pipelines

* make web-ci-pipeline as a template

* make web-ci-pipeline as a template

* make web-ci-pipeline as a template

* make web-ci-pipeline as a template

* change a paramter name checking a pipeline

* make a pool name changable for react native pipeline

* disable code sign validation for react native

* fix react native package.json publish

* fix indentation

* remove unnecessary comment

* test onnxruntime-common package publish

* ts and js files use lf as eol for windows

* use Linux style of ending line break

* change newLine at only tsconfig.json

* restore a commented code

* fix git restore directory for npm packaging

* fix a typo

* force eol to lf on windows for js directory in CI

js/common/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "dist/lib",
     "esModuleInterop": false,
-    "noUnusedParameters": true,
+    "noUnusedParameters": true
   },
   "include": ["lib"]
 }

js/node/package.json

@@ -55,4 +55,4 @@
   ], 
   "types": "dist/index.d.ts", 
   "description": "ONNXRuntime Node.js binding"
-}
+}

js/react_native/e2e/ios/Podfile

@@ -13,7 +13,6 @@ target 'OnnxruntimeModuleExample' do
   if File.exist?('../../local_pods/onnxruntime-mobile-c.zip')
     pod 'onnxruntime-mobile-c', :podspec => '../../onnxruntime-mobile-c.podspec'
   end
-  #pod 'onnxruntime-react-native', :path => '../node_modules/onnxruntime-react-native'
   pod 'onnxruntime-react-native', :path => '../..'
 
   inherit! :search_paths

js/react_native/tsconfig.json

@@ -22,6 +22,7 @@
     "resolveJsonModule": true,
     "skipLibCheck": true,
     "strict": true,
-    "target": "esnext"
+    "target": "esnext",
+    "newLine": "LF"
   }
 }

js/tsconfig.json

@@ -16,6 +16,7 @@
     "strictNullChecks": true,
     "pretty": true,
     "allowUnreachableCode": false,
-    "incremental": true
+    "incremental": true,
+    "newLine": "LF"
   }
 }

js/web/package.json

@@ -82,4 +82,4 @@
   "main": "dist/ort-web.node.js", 
   "types": "./types/lib/index.d.ts", 
   "description": "A Javascript library for running ONNX models on browsers"
-}
+}

tools/ci_build/github/azure-pipelines/mac-react-native-ci-pipeline.yml

@@ -11,6 +11,7 @@ parameters:
 
 variables:
   build_config: Release
+  pool_name: '$(PoolName)'
   ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
     npm_packaging_mode: 'dev'
   ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}:
@@ -28,11 +29,13 @@ jobs:
     includedOpsConfig: '$(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config'
     artifactName: 'onnxruntime-android-mobile-aar'
     job_name_suffix: 'For_React_Native'
-    pool_name: 'Linux-CPU-2019'
+    pool_name: '${{variables.pool_name}}'
 
 - job: ReactNative_CI
   pool:
     vmImage: 'macOS-11'
+  variables:
+    runCodesignValidationInjection: false
   dependsOn:
   - Android_Java_API_AAR_Packaging_For_React_Native
   timeoutInMinutes: 120
@@ -173,38 +176,15 @@ jobs:
   - task: PowerShell@2
     inputs:
       filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
-      arguments: '"${{variables.npm_packaging_mode}}" $(Build.SourcesDirectory) react_native'
+      arguments: '"-dev.$(Get-Date -Format yyyyMMdd)-$(git rev-parse --short HEAD)" $(Build.SourcesDirectory) react_native'
       workingDirectory: '$(Build.SourcesDirectory)'
       errorActionPreference: stop
     displayName: Pack NPM packages
 
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)/js/common
-      contents: onnxruntime-common*.tgz
-      targetFolder: $(Build.SourcesDirectory)/js/react_native/e2e
-    displayName: Copy onnxruntime-common npm package to React Native e2e directory
-
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)/js/react_native
-      contents: onnxruntime-react-native*.tgz
-      targetFolder: $(Build.SourcesDirectory)/js/react_native/e2e
-    displayName: Copy onnxruntime-react-native npm package to React Native e2e directory
-
-  # 'dev' npm_packaging_mode doesn't create npm package when onnxruntime-common is not changed.
-  # since e2e requires onnxruntime-common package, it builds and move it to e2e directory.
   - script: |
-      if [ ! -f $(Build.SourcesDirectory)/js/common/*.tgz ]
+      mv $(Build.SourcesDirectory)/js/common/onnxruntime-common*.tgz onnxruntime-common.tgz
-      then
-        pushd $(Build.SourcesDirectory)/js/common
-        npm pack
-        mv *.tgz $(Build.SourcesDirectory)/js/react_native/e2e
-        popd
-      fi
-      mv onnxruntime-common*.tgz onnxruntime-common.tgz
       yarn add --no-lockfile file:./onnxruntime-common.tgz
-      mv onnxruntime-react-native*.tgz onnxruntime-react-native.tgz
+      mv $(Build.SourcesDirectory)/js/react_native/onnxruntime-react-native*.tgz onnxruntime-react-native.tgz
       yarn add --no-lockfile file:./onnxruntime-react-native.tgz
       yarn
     workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e'
@@ -278,8 +258,18 @@ jobs:
 
   - script: |
       git restore .
-    workingDirectory: '$(Build.SourcesDirectory)/js/react_native'
+      cd react_native
-    displayName: Restore git changes for e2e tests
+      yarn prepack-rel
+    workingDirectory: '$(Build.SourcesDirectory)/js'
+    displayName: Restore git changes and prepack for npm publish
+
+  - task: PowerShell@2
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1'
+      arguments: '"${{variables.npm_packaging_mode}}" $(Build.SourcesDirectory) react_native'
+      workingDirectory: '$(Build.SourcesDirectory)'
+      errorActionPreference: stop
+    displayName: Pack NPM packages
 
   - task: CopyFiles@2
     inputs:

tools/ci_build/github/azure-pipelines/templates/web-ci.yml

@@ -0,0 +1,92 @@
+parameters:
+- name: NpmPackagingMode
+  displayName: 'NPM packages publish configuration'
+  type: string
+  default: 'dev'
+- name: IsReleasePipeline
+  displayName: 'Npm packaging pipeline'
+  type: boolean
+  default: false
+- name: PoolName
+  displayName: 'Agent pool name'
+  type: string
+  default: 'Win-CPU-2019'
+
+stages:
+- stage: Extract_commit
+  jobs:
+  - job: Extract_commit
+    pool: ${{ parameters.PoolName }}
+    variables:
+      runCodesignValidationInjection: false
+    timeoutInMinutes: 30
+    workspace:
+      clean: all
+    steps:
+    - checkout: none
+      fetchDepth: 1
+      submodules: false
+    - script: |
+        echo.$(Build.SourceVersion)
+        echo.$(Build.SourceVersion)>$(Build.ArtifactStagingDirectory)\__commit.txt
+    - task: PublishPipelineArtifact@0
+      displayName: 'Publish Pipeline Artifact'
+      inputs:
+        artifactName: '__commit'
+        targetPath: '$(Build.ArtifactStagingDirectory)'
+
+- stage: Build_wasm_Debug
+  dependsOn: Extract_commit
+  jobs:
+  - template: win-wasm-ci.yml
+    parameters:
+      CommitOverride: true
+      BuildConfig: 'Debug'
+      ExtraBuildArgs: '$(ExtraBuildArgs)'
+      PoolName: ${{ parameters.PoolName }}
+
+- stage: Build_web_Debug
+  dependsOn: Build_wasm_Debug
+  jobs:
+  - template: win-web-ci.yml
+    parameters:
+      CommitOverride: true
+      BuildConfig: 'Debug'
+      NpmPackagingMode: ${{ parameters.NpmPackagingMode }}
+      PoolName: ${{ parameters.PoolName }}
+
+- stage: Build_wasm_Release
+  dependsOn: Extract_commit
+  jobs:
+  - template: win-wasm-ci.yml
+    parameters:
+      CommitOverride: true
+      BuildConfig: 'Release'
+      ExtraBuildArgs: '--skip_tests --disable_wasm_exception_catching --disable_rtti $(ExtraBuildArgs)'
+      PoolName: ${{ parameters.PoolName }}
+
+- stage: Build_web_Release
+  dependsOn: Build_wasm_Release
+  jobs:
+  - template: win-web-ci.yml
+    parameters:
+      CommitOverride: true
+      BuildConfig: 'Release'
+      NpmPackagingMode: ${{ parameters.NpmPackagingMode }}
+      PoolName: ${{ parameters.PoolName }}
+
+- ${{ if ne(parameters.IsReleasePipeline, 'true') }}:
+  - stage: Test_web_BrowserStack
+    dependsOn: Build_wasm_Release
+    jobs:
+    - template: web-browserstack-ci.yml
+      parameters:
+        CommitOverride: true
+
+- ${{ if ne(parameters.IsReleasePipeline, 'true') }}:
+  - stage: Test_web_MultiBrowsers
+    dependsOn: Build_wasm_Release
+    jobs:
+    - template: win-web-multi-browsers.yml
+      parameters:
+        CommitOverride: true

tools/ci_build/github/azure-pipelines/templates/win-wasm-ci.yml

@@ -11,14 +11,19 @@ parameters:
   type: string
   default: ''
 
+- name: PoolName
+  type: string
+  default: 'Win-CPU-2019'
+
 jobs:
 - job: build_WASM
-  pool: 'Win-CPU-2019'
+  pool: ${{ parameters.PoolName }}
 
   variables:
     EnvSetupScript: setup_env.bat
     buildArch: x64
     CommonBuildArgs: '--config ${{ parameters.BuildConfig }} --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wasm --emsdk_version releases-upstream-2ddc66235392b37e5b33477fd86cbe01a14b8aa2-64bit ${{ parameters.ExtraBuildArgs }}'
+    runCodesignValidationInjection: false
   timeoutInMinutes: 120
   workspace:
     clean: all

tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml

@@ -11,10 +11,16 @@ parameters:
   type: string
   default: 'dev'
 
+- name: PoolName
+  type: string
+  default: 'Win-CPU-2019'
+
 jobs:
 - job: build_onnxruntime_web
-  pool: 'Win-CPU-2019'
+  pool: ${{ parameters.PoolName }}
 
+  variables:
+    runCodesignValidationInjection: false
   timeoutInMinutes: 30
   workspace:
     clean: all
@@ -44,6 +50,14 @@ jobs:
      git submodule update --init -- cmake\external\onnx
     workingDirectory: '$(Build.SourcesDirectory)'
     displayName: 'Checkout submodule onnx'
+  - script: |
+     echo.>>.gitattributes
+     echo /js/** text=auto eol=lf>>.gitattributes
+     rd /s /q js
+     git checkout -- js/**
+     git checkout -- .gitattributes
+    workingDirectory: '$(Build.SourcesDirectory)'
+    displayName: 'Testing: force EOL to lf on windows for /js/**' 
   - task: NodeTool@0
     inputs:
       versionSpec: '16.x'
@@ -93,7 +107,7 @@ jobs:
     workingDirectory: '$(Build.SourcesDirectory)\js'
     displayName: 'Clang-format'
   - script: |
-     node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)"
+     node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)"
     workingDirectory: '$(Build.SourcesDirectory)\js'
     displayName: 'Check unformatted files'
   - script: |
@@ -101,7 +115,7 @@ jobs:
     workingDirectory: '$(Build.SourcesDirectory)\js\web'
     displayName: 'Generating documents'
   - script: |
-     node -e "a=require('child_process').execSync('git ls-files -m').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)"
+     node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)"
     workingDirectory: '$(Build.SourcesDirectory)\js\web'
     displayName: 'Check out of dated documents'
   - task: PowerShell@2

tools/ci_build/github/azure-pipelines/web-ci-pipeline.yml

@@ -24,72 +24,8 @@ variables:
     NpmPackagingMode: '$(VersionSuffix)'
 
 stages:
-- stage: Extract_commit
+- template: templates/web-ci.yml
-  jobs:
+  parameters:
-  - job: Extract_commit
+    NpmPackagingMode: ${{ variables.NpmPackagingMode }}
-    pool: 'Win-CPU-2019'
+    IsReleasePipeline: false
-    timeoutInMinutes: 30
+    PoolName: 'Win-CPU-2019'
-    workspace:
-      clean: all
-    steps:
-    - checkout: none
-      fetchDepth: 1
-      submodules: false
-    - script: |
-        echo.$(Build.SourceVersion)
-        echo.$(Build.SourceVersion)>$(Build.ArtifactStagingDirectory)\__commit.txt
-    - task: PublishPipelineArtifact@0
-      displayName: 'Publish Pipeline Artifact'
-      inputs:
-        artifactName: '__commit'
-        targetPath: '$(Build.ArtifactStagingDirectory)'
-
-- stage: Build_wasm_Debug
-  dependsOn: Extract_commit
-  jobs:
-  - template: templates/win-wasm-ci.yml
-    parameters:
-      CommitOverride: true
-      BuildConfig: 'Debug'
-      ExtraBuildArgs: '$(ExtraBuildArgs)'
-
-- stage: Build_web_Debug
-  dependsOn: Build_wasm_Debug
-  jobs:
-  - template: templates/win-web-ci.yml
-    parameters:
-      CommitOverride: true
-      BuildConfig: 'Debug'
-      NpmPackagingMode: '$(NpmPackagingMode)'
-
-- stage: Build_wasm_Release
-  dependsOn: Extract_commit
-  jobs:
-  - template: templates/win-wasm-ci.yml
-    parameters:
-      CommitOverride: true
-      BuildConfig: 'Release'
-      ExtraBuildArgs: '--skip_tests --disable_wasm_exception_catching --disable_rtti $(ExtraBuildArgs)'
-
-- stage: Build_web_Release
-  dependsOn: Build_wasm_Release
-  jobs:
-  - template: templates/win-web-ci.yml
-    parameters:
-      CommitOverride: true
-      BuildConfig: 'Release'
-      NpmPackagingMode: '$(NpmPackagingMode)'
-
-- stage: Test_web_BrowserStack
-  dependsOn: Build_wasm_Release
-  jobs:
-  - template: templates/web-browserstack-ci.yml
-    parameters:
-      CommitOverride: true
-
-- stage: Test_web_MultiBrowsers
-  dependsOn: Build_wasm_Release
-  jobs:
-  - template: templates/win-web-multi-browsers.yml
-    parameters:
-      CommitOverride: true

tools/ci_build/github/azure-pipelines/web-packaging-pipeline.yml

@@ -0,0 +1,31 @@
+parameters:
+- name: NpmPublish
+  displayName: 'NPM packages publish configuration'
+  type: string
+  values:
+  - 'nightly (@dev)'
+  - 'release candidate (@rc)'
+  - 'production (@latest)'
+  - 'custom'
+  default: 'nightly (@dev)'
+
+variables:
+  # pipeline should define the following varaibles
+  #   ExtraBuildArgs
+  #   VersionSuffix
+
+  ${{ if eq(parameters.NpmPublish, 'nightly (@dev)') }}:
+    NpmPackagingMode: 'dev'
+  ${{ if eq(parameters.NpmPublish, 'release candidate (@rc)') }}:
+    NpmPackagingMode: 'rc'
+  ${{ if eq(parameters.NpmPublish, 'production (@latest)') }}:
+    NpmPackagingMode: 'release'
+  ${{ if eq(parameters.NpmPublish, 'custom') }}:
+    NpmPackagingMode: '$(VersionSuffix)'
+
+stages:
+- template: templates/web-ci.yml
+  parameters:
+    NpmPackagingMode: ${{ variables.NpmPackagingMode }}
+    IsReleasePipeline: true
+    PoolName: 'Win-CPU-2021'