From a272a75cd1a926ef17aa4bd2a550921e1b0dce80 Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Wed, 2 Jun 2021 17:49:12 -0700 Subject: [PATCH] [js/web] allow pull wasm artifacts from CI (#7886) * [js/web] allow pull wasm artifacts from CI * resolve comments --- js/README.md | 56 +++++--- js/web/package-lock.json | 33 +++++ js/web/package.json | 2 + js/web/script/pull-prebuilt-wasm-artifacts.ts | 127 ++++++++++++++++++ 4 files changed, 197 insertions(+), 21 deletions(-) create mode 100644 js/web/script/pull-prebuilt-wasm-artifacts.ts diff --git a/js/README.md b/js/README.md index 177cba8b1e..36194296df 100644 --- a/js/README.md +++ b/js/README.md @@ -145,39 +145,53 @@ This project is a library for running ONNX models on browsers. It is the success 2. in `/js/common/`, run `npm ci`. 3. in `/js/web/`, run `npm ci`. -2. ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it. Please see steps described as below.)~~ +2. Prepare ONNX Runtime WebAssembly artifacts. - in `/`, run one of the following commands to build WebAssembly: + You can either use the prebuilt artifacts or build it by yourself. - ```sh - # In windows, use 'build' to replace './build.sh' + - Setup by script. - # The following command build debug. - ./build.sh --build_wasm + In `/js/web/`, run `npm run pull:wasm` to pull WebAssembly artifacts for latest master branch from CI pipeline. - # The following command build debug with debug info. - ./build.sh --build_wasm --skip_tests --enable_wasm_debug_info + - Download artifacts from pipeline manually. - # The following command build release. - ./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti - ``` + you can download prebuilt WebAssembly artifacts from [Windows WebAssembly CI Pipeline](https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=161&_a=summary). Select a build, download artifacts "Release_ort-wasm" and "Release_ort-wasm-threaded" and unzip. See instructions below to put files into destination folders. - To build with multi-thread support, append flag `--enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step. + - Build WebAssembly artifacts. - NOTE: You can also find latest build artifacts on [Windows WebAssembly CI Pipeline](https://dev.azure.com/onnxruntime/onnxruntime/_build?definitionId=161&_a=summary&repositoryFilter=1&branchFilter=4%2C4%2C4%2C4%2C4%2C4). Choose any build for master branch, download artifacts "Release_ort-wasm" and "Release_ort-wasm-threaded" and unzip. + 1. Build ONNX Runtime WebAssembly -3. Copy following files from build output folder to `/js/web/dist/`: + ~~Follow [instructions](https://www.onnxruntime.ai/docs/how-to/build.html#apis-and-language-bindings) for building ONNX Runtime WebAssembly. (TODO: document is not ready. we are working on it. Please see steps described as below.)~~ - - ort-wasm.wasm - - ort-wasm-threaded.wasm (build with flag '--enable_wasm_threads') + in `/`, run one of the following commands to build WebAssembly: -4. Copy following files from build output folder to `/js/web/lib/wasm/binding/`: + ```sh + # In windows, use 'build' to replace './build.sh' - - ort-wasm.js - - ort-wasm-threaded.js (build with flag '--enable_wasm_threads') - - ort-wasm-threaded.worker.js (build with flag '--enable_wasm_threads') + # The following command build debug. + ./build.sh --build_wasm -5. Use following command in folder `/js/web` to build: + # The following command build debug with debug info. + ./build.sh --build_wasm --skip_tests --enable_wasm_debug_info + + # The following command build release. + ./build.sh --config Release --build_wasm --skip_tests --disable_wasm_exception_catching --disable_rtti + ``` + + To build with multi-thread support, append flag `--enable_wasm_threads` to the command. Make sure to build both single-thread and multi-thread before next step. + + 2. Copy following files from build output folder to `/js/web/dist/`: + + - ort-wasm.wasm + - ort-wasm-threaded.wasm (build with flag '--enable_wasm_threads') + + 3. Copy following files from build output folder to `/js/web/lib/wasm/binding/`: + + - ort-wasm.js + - ort-wasm-threaded.js (build with flag '--enable_wasm_threads') + - ort-wasm-threaded.worker.js (build with flag '--enable_wasm_threads') + +3. Use following command in folder `/js/web` to build: ``` npm run build ``` diff --git a/js/web/package-lock.json b/js/web/package-lock.json index 755aa33a89..2796479124 100644 --- a/js/web/package-lock.json +++ b/js/web/package-lock.json @@ -2442,6 +2442,12 @@ "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", "dev": true }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", + "dev": true + }, "import-local": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", @@ -2792,6 +2798,18 @@ "universalify": "^2.0.0" } }, + "jszip": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, "karma": { "version": "6.3.2", "resolved": "https://registry.npmjs.org/karma/-/karma-6.3.2.tgz", @@ -2953,6 +2971,15 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, "loader-runner": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.2.0.tgz", @@ -4133,6 +4160,12 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "dev": true + }, "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", diff --git a/js/web/package.json b/js/web/package.json index 0c3b0e1de3..b9df5f4d2c 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -17,6 +17,7 @@ "prepare": "tsc", "build": "node ./script/build", "build:doc": "node ./script/generate-operator-md", + "pull:wasm": "node ./script/pull-prebuilt-wasm-artifacts", "test": "node ./script/prepare-test-data && node ./script/test-runner-cli", "test:e2e": "node ./test/e2e/run", "prepack": "node ./script/prepack" @@ -46,6 +47,7 @@ "electron": "^12.0.2", "fs-extra": "^9.1.0", "globby": "^11.0.3", + "jszip": "^3.6.0", "karma": "^6.3.2", "karma-browserstack-launcher": "^1.6.0", "karma-chai": "^0.1.0", diff --git a/js/web/script/pull-prebuilt-wasm-artifacts.ts b/js/web/script/pull-prebuilt-wasm-artifacts.ts new file mode 100644 index 0000000000..fbb2dafa25 --- /dev/null +++ b/js/web/script/pull-prebuilt-wasm-artifacts.ts @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// +// This script is used to download WebAssembly build artifacts from CI pipeline. +// +// The goal of this script is to save time for ORT Web developers. For most TypeScript tasks, there is no change in the +// WebAssembly side, so there is no need to rebuild WebAssembly. +// +// It performs the following operations: +// 1. query build ID for latest successful build on master branch +// 2. query download URL of build artifacts +// 3. download and unzip the files to folders +// + +import fs from 'fs'; +import https from 'https'; +import jszip from 'jszip'; +import path from 'path'; + +function downloadJson(url: string, onSuccess: (data: any) => void) { + https.get(url, res => { + const {statusCode} = res; + const contentType = res.headers['content-type']; + + if (statusCode !== 200) { + throw new Error(`Failed to download build list. HTTP status code = ${statusCode}`); + } + if (!contentType || !/^application\/json/.test(contentType)) { + throw new Error(`unexpected content type: ${contentType}`); + } + res.setEncoding('utf8'); + let rawData = ''; + res.on('data', (chunk) => { + rawData += chunk; + }); + res.on('end', () => { + onSuccess(JSON.parse(rawData)); + }); + }); +} + +function downloadZip(url: string, onSuccess: (data: Buffer) => void) { + https.get(url, res => { + const {statusCode} = res; + const contentType = res.headers['content-type']; + + if (statusCode !== 200) { + throw new Error(`Failed to download build list. HTTP status code = ${statusCode}`); + } + if (!contentType || !/^application\/zip/.test(contentType)) { + throw new Error(`unexpected content type: ${contentType}`); + } + + const chunks: Buffer[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + }); + res.on('end', () => { + onSuccess(Buffer.concat(chunks)); + }); + }); +} + +function extractFile(zip: jszip, folder: string, file: string, artifactName: string) { + zip.file(`${artifactName}/${file}`)!.nodeStream() + .pipe(fs.createWriteStream(path.join(folder, file))) + .on('finish', () => { + console.log('# file downloaded and extracted: ' + file); + }); +} + +console.log('=== Start to pull WebAssembly artifacts from CI ==='); + +// API reference: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list +downloadJson( + 'https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/builds?api-version=6.1-preview.6' + + '&definitions=161' + + '&resultFilter=succeeded' + + '&$top=1' + + '&repositoryId=Microsoft/onnxruntime' + + '&repositoryType=GitHub' + + '&branchName=refs/heads/master', + data => { + const buildId = data.value[0].id; + + console.log(`=== Found latest master build : ${buildId} ===`); + + // API reference: https://docs.microsoft.com/en-us/rest/api/azure/devops/build/artifacts/get%20artifact + downloadJson( + `https://dev.azure.com/onnxruntime/onnxruntime/_apis/build/builds/${ + buildId}/artifacts?api-version=6.1-preview.5`, + data => { + let ortWasmZipLink, ortWasmThreadedZipLink; + for (const v of data.value) { + if (v.name === 'Release_ort-wasm') { + ortWasmZipLink = v.resource.downloadUrl; + } + if (v.name === 'Release_ort-wasm-threaded') { + ortWasmThreadedZipLink = v.resource.downloadUrl; + } + } + + console.log('=== Ready to download zip files ==='); + + const WASM_FOLDER = path.join(__dirname, '../dist'); + if (!fs.existsSync(WASM_FOLDER)) { + fs.mkdirSync(WASM_FOLDER); + } + const JS_FOLDER = path.join(__dirname, '../lib/wasm/binding'); + + downloadZip(ortWasmZipLink, buffer => { + void jszip.loadAsync(buffer).then(zip => { + extractFile(zip, JS_FOLDER, 'ort-wasm.js', 'Release_ort-wasm'); + extractFile(zip, WASM_FOLDER, 'ort-wasm.wasm', 'Release_ort-wasm'); + }); + }); + + downloadZip(ortWasmThreadedZipLink, buffer => { + void jszip.loadAsync(buffer).then(zip => { + extractFile(zip, JS_FOLDER, 'ort-wasm-threaded.js', 'Release_ort-wasm-threaded'); + extractFile(zip, JS_FOLDER, 'ort-wasm-threaded.worker.js', 'Release_ort-wasm-threaded'); + extractFile(zip, WASM_FOLDER, 'ort-wasm-threaded.wasm', 'Release_ort-wasm-threaded'); + }); + }); + }); + });