[Node.js binding] Allow installation to download CUDA binaries via script (#20364)

### Description
Currently we try to include all prebuilt binaries into the NPM packages.
This was working until we added libonnxruntime_providers_cuda.so
(>400MB) into the NPM package. The NPM registry refuses to accept new
package publishment because the file is too large.

To make the new NPM package working, we have to remove the large file
from the package, and add a new script on package installation. This
script will try to dynamically install onnxruntime CUDA dynamic library
for Linux/x64.

js/node/.gitignore

@@ -6,4 +6,5 @@ node_modules/
 
 /lib/**/*.js
 /script/**/*.js
+!/script/install.js
 /test/**/*.js

js/node/.npmignore

@@ -1,6 +1,7 @@
 /.vscode/
 /build/
 /script/
+!/script/install.js
 /src/
 /test/
 

js/node/README.md

@@ -14,13 +14,17 @@ Refer to [ONNX Runtime JavaScript examples](https://github.com/microsoft/onnxrun
 
 ## Requirements
 
-ONNXRuntime works on Node.js v12.x+ or Electron v5.x+.
+ONNXRuntime works on Node.js v16.x+ (recommend v18.x+) or Electron v15.x+ (recommend v28.x+).
 
-Following platforms are supported with pre-built binaries:
+The following table lists the supported versions of ONNX Runtime Node.js binding provided with pre-built binaries.
 
-- Windows x64 CPU NAPI_v3
-- Linux x64 CPU NAPI_v3
-- MacOS x64 CPU NAPI_v3
+| EPs/Platforms | Windows x64 | Windows arm64 | Linux x64         | Linux arm64 | MacOS x64 | MacOS arm64 |
+| ------------- | ----------- | ------------- | ----------------- | ----------- | --------- | ----------- |
+| CPU           | ✔️          | ✔️            | ✔️                | ✔️          | ✔️        | ✔️          |
+| DirectML      | ✔️          | ✔️            | ❌                | ❌          | ❌        | ❌          |
+| CUDA          | ❌          | ❌            | ✔️<sup>\[1]</sup> | ❌          | ❌        | ❌          |
+
+- \[1]: CUDA v11.8.
 
 To use on platforms without pre-built binaries, you can build Node.js binding from source and consume it by `npm install <onnxruntime_repo_root>/js/node/`. See also [instructions](https://onnxruntime.ai/docs/build/inferencing.html#apis-and-language-bindings) for building ONNX Runtime Node.js binding locally.
 
@@ -28,6 +32,20 @@ To use on platforms without pre-built binaries, you can build Node.js binding fr
 
 Right now, the Windows version supports only the DML provider. Linux x64 can use CUDA and TensorRT.
 
+## CUDA EP Installation
+
+To use CUDA EP, you need to install the CUDA EP binaries. By default, the CUDA EP binaries are installed automatically when you install the package. If you want to skip the installation, you can pass the `--onnxruntime-node-install-cuda=skip` flag to the installation command.
+
+```
+npm install onnxruntime-node --onnxruntime-node-install-cuda=skip
+```
+
+You can also use this flag to specify the version of the CUDA: (v11 or v12)
+
+```
+npm install onnxruntime-node --onnxruntime-node-install-cuda=v12
+```
+
 ## License
 
 License information can be found [here](https://github.com/microsoft/onnxruntime/blob/main/README.md#license).

js/node/package.json

@@ -15,9 +15,11 @@
   },
   "version": "1.18.0",
   "dependencies": {
-    "onnxruntime-common": "file:../common"
+    "onnxruntime-common": "file:../common",
+    "tar": "^7.0.1"
   },
   "scripts": {
+    "postinstall": "node ./script/install",
     "buildr": "tsc && node ./script/build --config=RelWithDebInfo",
     "preprepare": "node -e \"require('node:fs').copyFileSync('./node_modules/long/index.d.ts', './node_modules/long/umd/index.d.ts')\"",
     "prepare": "tsc --build script test .",

js/node/script/install.js

@@ -0,0 +1,132 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+// This script is written in JavaScript. This is because it is used in "install" script in package.json, which is called
+// when the package is installed either as a dependency or from "npm ci"/"npm install" without parameters. TypeScript is
+// not always available.
+
+// The purpose of this script is to download the required binaries for the platform and architecture.
+// Currently, most of the binaries are already bundled in the package, except for the following:
+// - Linux/x64/CUDA 11
+// - Linux/x64/CUDA 12
+//
+// The CUDA binaries are not bundled because they are too large to be allowed in the npm registry. Instead, they are
+// downloaded from the GitHub release page of ONNX Runtime. The script will download the binaries if they are not
+// already present in the package.
+
+// Step.1: Check if we should exit early
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const tar = require('tar');
+const {Readable} = require('stream');
+
+// commandline flag:
+// --onnxruntime-node-install-cuda         Force install the CUDA EP binaries. Try to detect the CUDA version.
+// --onnxruntime-node-install-cuda=v11     Force install the CUDA EP binaries for CUDA 11.
+// --onnxruntime-node-install-cuda=v12     Force install the CUDA EP binaries for CUDA 12.
+// --onnxruntime-node-install-cuda=skip    Skip the installation of the CUDA EP binaries.
+//
+// If the flag is not provided, the script will only install the CUDA EP binaries when:
+// - The platform is Linux/x64.
+// - The binaries are not already present in the package.
+// - The installation is not a local install (when used inside ONNX Runtime repo).
+//
+const INSTALL_CUDA_FLAG = parseInstallCudaFlag();
+const NO_INSTALL = INSTALL_CUDA_FLAG === 'skip';
+const FORCE_INSTALL = !NO_INSTALL && INSTALL_CUDA_FLAG;
+
+const IS_LINUX_X64 = os.platform() === 'linux' && os.arch() === 'x64';
+const BIN_FOLDER = path.join(__dirname, '..', 'bin/napi-v3/linux/x64');
+const BIN_FOLDER_EXISTS = fs.existsSync(BIN_FOLDER);
+const CUDA_DLL_EXISTS = fs.existsSync(path.join(BIN_FOLDER, 'libonnxruntime_providers_cuda.so'));
+const ORT_VERSION = require('../package.json').version;
+
+const npm_config_local_prefix = process.env.npm_config_local_prefix;
+const npm_package_json = process.env.npm_package_json;
+const SKIP_LOCAL_INSTALL =
+    npm_config_local_prefix && npm_package_json && path.dirname(npm_package_json) === npm_config_local_prefix;
+
+const shouldInstall = FORCE_INSTALL || (!SKIP_LOCAL_INSTALL && IS_LINUX_X64 && BIN_FOLDER_EXISTS && !CUDA_DLL_EXISTS);
+if (NO_INSTALL || !shouldInstall) {
+  process.exit(0);
+}
+
+// Step.2: Download the required binaries
+const artifactUrl = {
+  11: `https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-gpu-${
+      ORT_VERSION}.tgz`,
+  12: `https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-cuda12-${
+      ORT_VERSION}.tgz`
+}[INSTALL_CUDA_FLAG || tryGetCudaVersion()];
+console.log(`Downloading "${artifactUrl}"...`);
+fetch(artifactUrl).then(res => {
+  if (!res.ok) {
+    throw new Error(`Failed to download the binaries: ${res.status} ${res.statusText}.
+
+Use "--onnxruntime-node-install-cuda=skip" to skip the installation. You will still be able to use ONNX Runtime, but the CUDA EP will not be available.`);
+  }
+
+  // Extract the binaries
+
+  const FILES = new Set([
+    'libonnxruntime_providers_tensorrt.so',
+    'libonnxruntime_providers_shared.so',
+    `libonnxruntime.so.${ORT_VERSION}`,
+    'libonnxruntime_providers_cuda.so',
+  ]);
+
+  Readable.fromWeb(res.body)
+      .pipe(tar.t())
+      .on('entry',
+          (entry) => {
+            const filename = path.basename(entry.path);
+            if (entry.type === 'File' && FILES.has(filename)) {
+              console.log(`Extracting "${filename}" to "${BIN_FOLDER}"...`);
+              entry.pipe(fs.createWriteStream(path.join(BIN_FOLDER, filename)));
+            }
+          })
+      .on('error', (err) => {
+        throw new Error(`Failed to extract the binaries: ${err.message}.
+
+Use "--onnxruntime-node-install-cuda=skip" to skip the installation. You will still be able to use ONNX Runtime, but the CUDA EP will not be available.`);
+      });
+});
+
+
+function tryGetCudaVersion() {
+  // Should only return 11 or 12.
+
+  // TODO: try to get the CUDA version from the system ( `nvcc --version` )
+
+  return 11;
+}
+
+function parseInstallCudaFlag() {
+  let flag = process.env.npm_config_onnxruntime_node_install_cuda;
+  if (!flag) {
+    for (let i = 0; i < process.argv.length; i++) {
+      if (process.argv[i].startsWith('--onnxruntime-node-install-cuda=')) {
+        flag = process.argv[i].split('=')[1];
+        break;
+      } else if (process.argv[i] === '--onnxruntime-node-install-cuda') {
+        flag = 'true';
+      }
+    }
+  }
+  switch (flag) {
+    case 'true':
+      return tryGetCudaVersion();
+    case 'v11':
+      return 11;
+    case 'v12':
+      return 12;
+    case 'skip':
+    case undefined:
+      return flag;
+    default:
+      throw new Error(`Invalid value for --onnxruntime-node-install-cuda: ${flag}`);
+  }
+}

tools/ci_build/github/azure-pipelines/nodejs/templates/test.yml

@@ -29,7 +29,7 @@ steps:
 
 - script: |
     npm init -y
-    npm install $(NpmPackageFilesForTest)
+    npm install $(NpmPackageFilesForTest) --onnxruntime-node-install-cuda=skip
     node -p "require('onnxruntime-node')"
   workingDirectory: '$(Build.BinariesDirectory)/e2e_test'
 

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -700,7 +700,7 @@ stages:
         SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-tensorrt\lib'
         Contents: |
           libonnxruntime.so.*
-          libonnxruntime_providers_*.so
+          libonnxruntime_providers_shared.so
         TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64'
     - task: CopyFiles@2
       displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64\'