[js/web] allow pull wasm artifacts from CI (#7886)

* [js/web] allow pull wasm artifacts from CI

* resolve comments

js/README.md

@@ -145,39 +145,53 @@ This project is a library for running ONNX models on browsers. It is the success
    2. in `<ORT_ROOT>/js/common/`, run `npm ci`.
    3. in `<ORT_ROOT>/js/web/`, run `npm ci`.
 
-2. ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it. Please see steps described as below.)~~
+2. Prepare ONNX Runtime WebAssembly artifacts.
 
-   in `<ORT_ROOT>/`, run one of the following commands to build WebAssembly:
+   You can either use the prebuilt artifacts or build it by yourself.
 
-   ```sh
-   # In windows, use 'build' to replace './build.sh'
+   - Setup by script.
 
-   # The following command build debug.
-   ./build.sh --build_wasm
+     In `<ORT_ROOT>/js/web/`, run `npm run pull:wasm` to pull WebAssembly artifacts for latest master branch from CI pipeline.
 
-   # The following command build debug with debug info.
-   ./build.sh --build_wasm --skip_tests --enable_wasm_debug_info
+   - Download artifacts from pipeline manually.
 
-   # The following command build release.
-   ./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti
-   ```
+     you can download prebuilt WebAssembly artifacts from [Windows WebAssembly CI Pipeline](https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=161&_a=summary). Select a build, download artifacts "Release_ort-wasm" and "Release_ort-wasm-threaded" and unzip. See instructions below to put files into destination folders.
 
-   To build with multi-thread support, append flag `--enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step.
+   - Build WebAssembly artifacts.
 
-   NOTE: You can also find latest build artifacts on [Windows WebAssembly CI Pipeline](https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=161&_a=summary&repositoryFilter=1&branchFilter=4%2C4%2C4%2C4%2C4%2C4). Choose any build for master branch, download artifacts "Release_ort-wasm" and "Release_ort-wasm-threaded" and unzip.
+     1. Build ONNX Runtime WebAssembly
 
-3. Copy following files from build output folder to `<ORT_ROOT>/js/web/dist/`:
+        ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it. Please see steps described as below.)~~
 
-   - ort-wasm.wasm
-   - ort-wasm-threaded.wasm (build with flag '--enable_wasm_threads')
+        in `<ORT_ROOT>/`, run one of the following commands to build WebAssembly:
 
-4. Copy following files from build output folder to `<ORT_ROOT>/js/web/lib/wasm/binding/`:
+        ```sh
+        # In windows, use 'build' to replace './build.sh'
 
-   - ort-wasm.js
-   - ort-wasm-threaded.js (build with flag '--enable_wasm_threads')
-   - ort-wasm-threaded.worker.js (build with flag '--enable_wasm_threads')
+        # The following command build debug.
+        ./build.sh --build_wasm
 
-5. Use following command in folder `<ORT_ROOT>/js/web` to build:
+        # The following command build debug with debug info.
+        ./build.sh --build_wasm --skip_tests --enable_wasm_debug_info
+
+        # The following command build release.
+        ./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti
+        ```
+
+        To build with multi-thread support, append flag `--enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step.
+
+     2. Copy following files from build output folder to `<ORT_ROOT>/js/web/dist/`:
+
+        - ort-wasm.wasm
+        - ort-wasm-threaded.wasm (build with flag '--enable_wasm_threads')
+
+     3. Copy following files from build output folder to `<ORT_ROOT>/js/web/lib/wasm/binding/`:
+
+        - ort-wasm.js
+        - ort-wasm-threaded.js (build with flag '--enable_wasm_threads')
+        - ort-wasm-threaded.worker.js (build with flag '--enable_wasm_threads')
+
+3. Use following command in folder `<ORT_ROOT>/js/web` to build:
    ```
    npm run build
    ```

js/web/package-lock.json

@@ -2442,6 +2442,12 @@
       "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
       "dev": true
     },
+    "immediate": {
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
+      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=",
+      "dev": true
+    },
     "import-local": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz",
@@ -2792,6 +2798,18 @@
         "universalify": "^2.0.0"
       }
     },
+    "jszip": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz",
+      "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==",
+      "dev": true,
+      "requires": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "set-immediate-shim": "~1.0.1"
+      }
+    },
     "karma": {
       "version": "6.3.2",
       "resolved": "https://registry.npmjs.org/karma/-/karma-6.3.2.tgz",
@@ -2953,6 +2971,15 @@
       "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
       "dev": true
     },
+    "lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dev": true,
+      "requires": {
+        "immediate": "~3.0.5"
+      }
+    },
     "loader-runner": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz",
@@ -4133,6 +4160,12 @@
       "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
       "dev": true
     },
+    "set-immediate-shim": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
+      "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=",
+      "dev": true
+    },
     "setimmediate": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",

js/web/package.json

@@ -17,6 +17,7 @@
     "prepare": "tsc",
     "build": "node ./script/build",
     "build:doc": "node ./script/generate-operator-md",
+    "pull:wasm": "node ./script/pull-prebuilt-wasm-artifacts",
     "test": "node ./script/prepare-test-data && node ./script/test-runner-cli",
     "test:e2e": "node ./test/e2e/run",
     "prepack": "node ./script/prepack"
@@ -46,6 +47,7 @@
     "electron": "^12.0.2",
     "fs-extra": "^9.1.0",
     "globby": "^11.0.3",
+    "jszip": "^3.6.0",
     "karma": "^6.3.2",
     "karma-browserstack-launcher": "^1.6.0",
     "karma-chai": "^0.1.0",

js/web/script/pull-prebuilt-wasm-artifacts.ts

@@ -0,0 +1,127 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+//
+// This script is used to download WebAssembly build artifacts from CI pipeline.
+//
+// The goal of this script is to save time for ORT Web developers. For most TypeScript tasks, there is no change in the
+// WebAssembly side, so there is no need to rebuild WebAssembly.
+//
+// It performs the following operations:
+// 1. query build ID for latest successful build on master branch
+// 2. query download URL of build artifacts
+// 3. download and unzip the files to folders
+//
+
+import fs from 'fs';
+import https from 'https';
+import jszip from 'jszip';
+import path from 'path';
+
+function downloadJson(url: string, onSuccess: (data: any) => void) {
+  https.get(url, res => {
+    const {statusCode} = res;
+    const contentType = res.headers['content-type'];
+
+    if (statusCode !== 200) {
+      throw new Error(`Failed to download build list. HTTP status code = ${statusCode}`);
+    }
+    if (!contentType || !/^application\/json/.test(contentType)) {
+      throw new Error(`unexpected content type: ${contentType}`);
+    }
+    res.setEncoding('utf8');
+    let rawData = '';
+    res.on('data', (chunk) => {
+      rawData += chunk;
+    });
+    res.on('end', () => {
+      onSuccess(JSON.parse(rawData));
+    });
+  });
+}
+
+function downloadZip(url: string, onSuccess: (data: Buffer) => void) {
+  https.get(url, res => {
+    const {statusCode} = res;
+    const contentType = res.headers['content-type'];
+
+    if (statusCode !== 200) {
+      throw new Error(`Failed to download build list. HTTP status code = ${statusCode}`);
+    }
+    if (!contentType || !/^application\/zip/.test(contentType)) {
+      throw new Error(`unexpected content type: ${contentType}`);
+    }
+
+    const chunks: Buffer[] = [];
+    res.on('data', (chunk) => {
+      chunks.push(chunk);
+    });
+    res.on('end', () => {
+      onSuccess(Buffer.concat(chunks));
+    });
+  });
+}
+
+function extractFile(zip: jszip, folder: string, file: string, artifactName: string) {
+  zip.file(`${artifactName}/${file}`)!.nodeStream()
+      .pipe(fs.createWriteStream(path.join(folder, file)))
+      .on('finish', () => {
+        console.log('# file downloaded and extracted: ' + file);
+      });
+}
+
+console.log('=== Start to pull WebAssembly artifacts from CI ===');
+
+// API reference: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list
+downloadJson(
+    'https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/builds?api-version=6.1-preview.6' +
+        '&definitions=161' +
+        '&resultFilter=succeeded' +
+        '&$top=1' +
+        '&repositoryId=Microsoft/onnxruntime' +
+        '&repositoryType=GitHub' +
+        '&branchName=refs/heads/master',
+    data => {
+      const buildId = data.value[0].id;
+
+      console.log(`=== Found latest master build : ${buildId} ===`);
+
+      // API reference: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/artifacts/get%20artifact
+      downloadJson(
+          `https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/builds/${
+              buildId}/artifacts?api-version=6.1-preview.5`,
+          data => {
+            let ortWasmZipLink, ortWasmThreadedZipLink;
+            for (const v of data.value) {
+              if (v.name === 'Release_ort-wasm') {
+                ortWasmZipLink = v.resource.downloadUrl;
+              }
+              if (v.name === 'Release_ort-wasm-threaded') {
+                ortWasmThreadedZipLink = v.resource.downloadUrl;
+              }
+            }
+
+            console.log('=== Ready to download zip files ===');
+
+            const WASM_FOLDER = path.join(__dirname, '../dist');
+            if (!fs.existsSync(WASM_FOLDER)) {
+              fs.mkdirSync(WASM_FOLDER);
+            }
+            const JS_FOLDER = path.join(__dirname, '../lib/wasm/binding');
+
+            downloadZip(ortWasmZipLink, buffer => {
+              void jszip.loadAsync(buffer).then(zip => {
+                extractFile(zip, JS_FOLDER, 'ort-wasm.js', 'Release_ort-wasm');
+                extractFile(zip, WASM_FOLDER, 'ort-wasm.wasm', 'Release_ort-wasm');
+              });
+            });
+
+            downloadZip(ortWasmThreadedZipLink, buffer => {
+              void jszip.loadAsync(buffer).then(zip => {
+                extractFile(zip, JS_FOLDER, 'ort-wasm-threaded.js', 'Release_ort-wasm-threaded');
+                extractFile(zip, JS_FOLDER, 'ort-wasm-threaded.worker.js', 'Release_ort-wasm-threaded');
+                extractFile(zip, WASM_FOLDER, 'ort-wasm-threaded.wasm', 'Release_ort-wasm-threaded');
+              });
+            });
+          });
+    });