[node.js binding] aggregate binaries for multiple platforms in single NPM package (#9501)

cmake/onnxruntime_nodejs.cmake

@@ -41,8 +41,8 @@ add_custom_target(js_common_npm_ci ALL
     COMMENT "NPM install on /js/common")
 
 add_custom_target(nodejs_binding_wrapper ALL
-    COMMAND ${NPM_CLI} ci --ort-skip-build
-    COMMAND ${NPM_CLI} run build -- --onnxruntime-build-dir=${CMAKE_CURRENT_BINARY_DIR} --config=${CMAKE_BUILD_TYPE}
+    COMMAND ${NPM_CLI} ci
+    COMMAND ${NPM_CLI} run build -- --onnxruntime-build-dir=${CMAKE_CURRENT_BINARY_DIR} --config=${CMAKE_BUILD_TYPE} --arch=x64
     WORKING_DIRECTORY ${JS_NODE_ROOT}
     COMMENT "Using cmake-js to build OnnxRuntime Node.js binding")
 add_dependencies(js_common_npm_ci js_npm_ci)

js/node/.gitignore

@@ -2,7 +2,6 @@ node_modules/
 /build/
 /bin/
 /dist/
-/prebuilds/
 /types/
 
 /lib/**/*.js

js/node/.npmignore

@@ -1,7 +1,5 @@
 /.vscode/
-/bin/
 /build/
-/prebuilds/
 /script/
 /src/
 /test/

js/node/package.json

@@ -20,33 +20,25 @@
     "rebuild": "tsc && node ./script/build --rebuild",
     "rebuildd": "tsc && node ./script/build --rebuild --config=Debug",
     "rebuildr": "tsc && node ./script/build --rebuild --config=RelWithDebInfo",
-    "install": "prebuild-install -r napi || (tsc --build script && node ./script/build)",
     "prepare": "tsc --build script && tsc",
     "test": "tsc --build test && mocha ./test/test-main",
     "prepack": "node ./script/prepack"
   },
   "dependencies": {
-    "onnxruntime-common": "file:../common",
-    "prebuild-install": "^6.1.2"
+    "onnxruntime-common": "file:../common"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.6",
-    "@types/klaw-sync": "^6.0.0",
     "@types/minimist": "1.2.1",
     "@types/mocha": "^8.2.2",
     "@types/node": "^14.14.37",
-    "@types/tar-stream": "^2.2.0",
     "cmake-js": "^6.1.0",
     "fs-extra": "^9.1.0",
-    "globby": "^11.0.2",
     "jsonc": "^2.0.0",
-    "klaw-sync": "^6.0.0",
     "minimist": "^1.2.5",
     "mocha": "^8.2.1",
     "node-addon-api": "^3.1.0",
     "onnx-proto": "^4.0.4",
-    "tar-stream": "^2.2.0",
-    "typedoc": "^0.20.25",
     "typescript": "^4.2.4"
   },
   "main": "dist/index.js",
@@ -63,4 +55,4 @@
       3
     ]
   }
-}
+}

js/node/script/build.ts

@@ -6,23 +6,22 @@ import * as fs from 'fs-extra';
 import minimist from 'minimist';
 import * as path from 'path';
 
-// NPM configs (parsed via 'npm install --xxx')
-
-// skip build on install. usually used in CI where build will be another step.
-const SKIP = !!process.env.npm_config_ort_skip_build;
-if (SKIP) {
-  process.exit(0);
-}
-
 // command line flags
 const buildArgs = minimist(process.argv.slice(2));
 
-// currently only support Debug, Release and RelWithDebInfo
+// --config=Debug|Release|RelWithDebInfo
 const CONFIG: 'Debug'|'Release'|'RelWithDebInfo' = buildArgs.config || 'RelWithDebInfo';
 if (CONFIG !== 'Debug' && CONFIG !== 'Release' && CONFIG !== 'RelWithDebInfo') {
   throw new Error(`unrecognized config: ${CONFIG}`);
 }
+// --arch=x64|ia32|arm64|arm
+const ARCH: 'x64'|'ia32'|'arm64'|'arm' = buildArgs.arch || 'x64';
+if (ARCH !== 'x64' && ARCH !== 'ia32' && ARCH !== 'arm64' && ARCH !== 'arm') {
+  throw new Error(`unrecognized architecture: ${ARCH}`);
+}
+// --onnxruntime-build-dir=
 const ONNXRUNTIME_BUILD_DIR = buildArgs['onnxruntime-build-dir'];
+// --rebuild
 const REBUILD = !!buildArgs.rebuild;
 
 // build path
@@ -42,7 +41,7 @@ if (REBUILD) {
 const command = CMAKE_JS_FULL_PATH;
 const args = [
   (REBUILD ? 'reconfigure' : 'configure'),
-  '--arch=x64',
+  `--arch=${ARCH}`,
   '--CDnapi_build_version=3',
   `--CDCMAKE_BUILD_TYPE=${CONFIG}`,
 ];

js/node/script/prepack.ts

@@ -2,10 +2,7 @@
 // Licensed under the MIT License.
 
 import * as fs from 'fs-extra';
-import klawSync from 'klaw-sync';
 import * as path from 'path';
-import {pack} from 'tar-stream';
-import * as zlib from 'zlib';
 
 function updatePackageJson() {
   const commonPackageJsonPath = path.join(__dirname, '..', '..', 'common', 'package.json');
@@ -19,53 +16,5 @@ function updatePackageJson() {
   console.log('=== finished updating package.json.');
 }
 
-function packPrebuild() {
-  // build path
-  const ROOT_FOLDER = path.join(__dirname, '..');
-  const BIN_FOLDER = path.join(ROOT_FOLDER, 'bin');
-  const PREBUILDS_FOLDER = path.join(ROOT_FOLDER, 'prebuilds');
-
-  // start pack
-  const tarName = `${process.env.npm_package_name}-v${process.env.npm_package_version}-napi-v3-${process.platform}-${
-      process.arch}.tar.gz`;
-  const tarPath = path.join(PREBUILDS_FOLDER, tarName);
-
-  const tarStream = pack();
-  fs.ensureDirSync(PREBUILDS_FOLDER);
-  const ws = fs.createWriteStream(tarPath);
-
-  tarStream.pipe(zlib.createGzip({level: 9})).pipe(ws);
-
-  // enumerate all files under BIN folder
-  const entries =
-      klawSync(BIN_FOLDER, {nodir: true}).map(i => ({
-                                                path: i.path,
-                                                name: path.relative(ROOT_FOLDER, i.path).replace(/\\/g, '/'),
-                                                size: i.stats.size,
-                                                // eslint-disable-next-line no-bitwise
-                                                mode: i.stats.mode | parseInt('444', 8) | parseInt('222', 8),
-                                                gid: i.stats.gid,
-                                                uid: i.stats.uid
-                                              }));
-
-  console.log(`=== start to pack prebuild: ${tarName}`);
-
-  function packNextFile(): void {
-    const nextEntry = entries.shift();
-    if (nextEntry) {
-      console.log(`  adding file: ${nextEntry.name}`);
-      const stream = tarStream.entry(nextEntry);
-      fs.createReadStream(nextEntry.path).pipe(stream).on('finish', packNextFile);
-    } else {
-      console.log('=== finished packing prebuild.');
-      tarStream.finalize();
-    }
-  }
-
-  packNextFile();
-}
-
 // update version of dependency "onnxruntime-common" before packing
 updatePackageJson();
-// pack prebuilds
-packPrebuild();

tools/ci_build/github/azure-pipelines/nodejs/cpu-esrp-pipeline.yml

@@ -121,25 +121,16 @@ jobs:
 
   - script: |
       del /Q $(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\x64\CodeSignSummary-*.*
-      call npm pack
-      copy $(Build.SourcesDirectory)\js\node\onnxruntime-*.tgz $(Build.ArtifactStagingDirectory)
-      xcopy /E /I $(Build.SourcesDirectory)\js\node\prebuilds $(Build.ArtifactStagingDirectory)\prebuilds
+      xcopy /E /I $(Build.SourcesDirectory)\js\node\bin $(Build.ArtifactStagingDirectory)\bin
     workingDirectory: '$(Build.SourcesDirectory)\js\node'
-    displayName: 'Create NPM Package'
+    displayName: 'Create Node.js Binding Artifact'
 
   - task: PublishPipelineArtifact@0
-    displayName: 'Publish Pipeline Artifact: drop-win32_x64'
+    displayName: 'Publish Pipeline Artifact: bin-win32_x64'
     inputs:
-      artifactName: 'drop-win32_x64'
+      artifactName: 'bin-win32_x64'
       targetPath: '$(Build.ArtifactStagingDirectory)'
 
-  # Put an unzipped version there to check if all the binaries are signed.
-  - script: |
-      7z x $(Build.ArtifactStagingDirectory)\prebuilds\onnxruntime-*.tar.gz
-      7z x $(Build.ArtifactStagingDirectory)\onnxruntime-*.tar
-    displayName: 'Unzip package to test'
-    workingDirectory: '$(Build.ArtifactStagingDirectory)'
-
   - task: DeleteFiles@1
     displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
     condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))
@@ -175,7 +166,7 @@ jobs:
 - template: ../templates/mac-ci.yml
   parameters:
     AgentPool : $(AgentPoolMacOS)
-    ArtifactName: 'drop-darwin_x64'
+    ArtifactName: 'bin-darwin_x64'
     JobName: 'Mac_CI_x64'
     BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_nodejs --config Release'
     DoNodejsPack : 'true'
@@ -201,19 +192,64 @@ jobs:
           mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \
           --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
           /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_nodejs \
-          --skip_submodule_sync --parallel --build_shared_lib && cd /onnxruntime_src/js/node && npm pack"
+          --skip_submodule_sync --parallel --build_shared_lib"
         workingDirectory: $(Build.SourcesDirectory)
+      displayName: 'Build and test'
     - script: |
        set -e -x
-       cp $(Build.SourcesDirectory)/js/node/onnxruntime-*.tgz $(Build.ArtifactStagingDirectory)
-       cp -R $(Build.SourcesDirectory)/js/node/prebuilds $(Build.ArtifactStagingDirectory)/prebuilds
-      displayName: 'Create Artifacts'
+       cp -R $(Build.SourcesDirectory)/js/node/bin $(Build.ArtifactStagingDirectory)/bin
+      displayName: 'Create Node.js Binding Artifact'
     - task: PublishPipelineArtifact@0
       displayName: 'Publish Pipeline Artifact'
       inputs:
-        artifactName: 'drop-linux_x64'
+        artifactName: 'bin-linux_x64'
         targetPath: '$(Build.ArtifactStagingDirectory)'
     - template: ../templates/component-governance-component-detection-steps.yml
       parameters :
         condition : 'succeeded'
     - template: ../templates/clean-agent-build-directory-step.yml
+
+- job: 'NPM_pack'
+  workspace:
+    clean: all
+  timeoutInMinutes: 120
+  dependsOn:
+  - Windows_CI_x64
+  - Mac_CI_x64
+  - Linux_CI_x64
+  pool: $(AgentPoolLinux)
+  steps:    
+    - template: ../templates/set-version-number-variables-step.yml
+    - template: ../templates/get-docker-image-steps.yml
+      parameters:
+        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
+        Context: tools/ci_build/github/linux/docker
+        DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
+        Repository: onnxruntimecpubuild
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        patterns: 'bin-*/**/*'
+        path: $(Pipeline.Workspace)/nodejs_binding_bin
+      displayName: 'Download Node.js binding artifacts'
+    - task: CmdLine@2
+      inputs:
+        script: find . -type f -follow -print | xargs ls -l
+        workingDirectory: $(Pipeline.Workspace)/nodejs_binding_bin
+      displayName: 'Validate file list of all binding binaries'
+    - task: CmdLine@2
+      inputs:
+        script: |
+          mkdir -p $HOME/.onnx && docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
+          --volume $(Pipeline.Workspace)/nodejs_binding_bin:/nodejs_binding_bin:ro -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "cd /onnxruntime_src/js && \
+          npm ci && cd /onnxruntime_src/js/common && npm ci && npm pack && cd /onnxruntime_src/js/node && npm ci && cp -r /nodejs_binding_bin/*/bin/ . && npm pack"
+        workingDirectory: $(Build.SourcesDirectory)
+    - script: |
+       set -e -x
+       cp $(Build.SourcesDirectory)/js/common/onnxruntime-*.tgz $(Build.ArtifactStagingDirectory)
+       cp $(Build.SourcesDirectory)/js/node/onnxruntime-*.tgz $(Build.ArtifactStagingDirectory)
+      displayName: 'Create Artifacts'
+    - task: PublishPipelineArtifact@0
+      displayName: 'Publish Pipeline Artifact'
+      inputs:
+        artifactName: 'NPM_packages'
+        targetPath: '$(Build.ArtifactStagingDirectory)'

tools/ci_build/github/azure-pipelines/templates/mac-ci.yml

@@ -77,11 +77,9 @@ jobs:
       #     Pattern: '*.dylib,*.node'
 
       - script: |
-         npm pack
-         cp $(Build.SourcesDirectory)/js/node/onnxruntime-*.tgz $(Build.ArtifactStagingDirectory)
-         cp -R $(Build.SourcesDirectory)/js/node/prebuilds $(Build.ArtifactStagingDirectory)/prebuilds
+         cp -R $(Build.SourcesDirectory)/js/node/bin $(Build.ArtifactStagingDirectory)/bin
         workingDirectory: '$(Build.SourcesDirectory)/js/node'
-        displayName: 'Create NPM Package'
+        displayName: 'Create Node.js Binding Artifact'
 
       - task: PublishPipelineArtifact@0
         displayName: 'Publish Pipeline Artifact: ${{ parameters.ArtifactName }}'