Add Node.js binding support to packaging pipeline (#9577)

This commit is contained in:
Yulong Wang 2021-11-05 15:29:40 -07:00 committed by GitHub
parent 1cbbafdbe0
commit c6fddb263f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 670 additions and 34 deletions

View file

@ -29,14 +29,34 @@ if(had_error)
endif()
# setup ARCH
if (APPLE AND CMAKE_OSX_ARCHITECTURES_LEN GREATER 1)
message(FATAL_ERROR "CMake.js does not support multi-architecture for macOS")
endif()
if (APPLE AND CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")
set(NODEJS_BINDING_ARCH arm64)
# elseif()
if (APPLE)
if (CMAKE_OSX_ARCHITECTURES_LEN GREATER 1)
message(FATAL_ERROR "CMake.js does not support multi-architecture for macOS")
endif()
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")
set(NODEJS_BINDING_ARCH arm64)
else()
set(NODEJS_BINDING_ARCH x64)
endif()
elseif (WIN32)
if (NOT MSVC)
message(FATAL_ERROR "Only support MSVC for building Node.js binding on Windows.")
endif()
if(onnxruntime_target_platform STREQUAL "ARM64")
set(NODEJS_BINDING_ARCH arm64)
elseif(onnxruntime_target_platform STREQUAL "x64")
set(NODEJS_BINDING_ARCH x64)
else()
message(FATAL_ERROR "Unsupported target platform for Node.js binding:" ${onnxruntime_target_platform})
endif()
else()
set(NODEJS_BINDING_ARCH x64)
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
set(NODEJS_BINDING_ARCH arm64)
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
set(NODEJS_BINDING_ARCH x64)
else()
message(FATAL_ERROR "Unsupported target platform for Node.js binding:" ${onnxruntime_target_platform})
endif()
endif()
if(NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
@ -51,11 +71,45 @@ add_custom_target(js_common_npm_ci ALL
WORKING_DIRECTORY ${JS_COMMON_ROOT}
COMMENT "NPM install on /js/common")
add_custom_target(nodejs_binding_wrapper ALL
COMMAND ${NPM_CLI} ci
COMMAND ${NPM_CLI} run build -- --onnxruntime-build-dir=${CMAKE_CURRENT_BINARY_DIR} --config=${CMAKE_BUILD_TYPE} --arch=${NODEJS_BINDING_ARCH}
WORKING_DIRECTORY ${JS_NODE_ROOT}
COMMENT "Using cmake-js to build OnnxRuntime Node.js binding")
if(WIN32 AND NODEJS_BINDING_ARCH STREQUAL "arm64")
# NOTE:
# Node.js for Windows ARM64 is not available in official release list.
# We uses node.lib for Node.js ARM64 v16.2.0 unofficial build from:
# https://unofficial-builds.nodejs.org/download/release/v16.2.0/win-arm64/
# The file should be downloaded and saved to ${CMAKE_CURRENT_BINARY_DIR}
#
# once Node.js for Windows ARM64 is in official release list and cmake-js supports Windows ARM64,
# we can remove this section.
include(FetchContent)
FetchContent_Declare(
nodejs_win_arm64_node_lib
URL https://unofficial-builds.nodejs.org/download/release/v16.2.0/win-arm64/node.lib
URL_HASH SHA256=decbf5c6f7e048af34add43da5eaa75b309340acc17b2670b8289a47dd751401
DOWNLOAD_NO_EXTRACT TRUE
)
FetchContent_MakeAvailable(nodejs_win_arm64_node_lib)
FetchContent_Declare(
nodejs_win_arm64_headers
URL https://unofficial-builds.nodejs.org/download/release/v16.2.0/node-v16.2.0-headers.tar.gz
URL_HASH SHA256=4f6a1e877c65ce31217b1c67f4fd90b9f9f4d3051e5ffabf151f43dd6edf7a34
)
FetchContent_MakeAvailable(nodejs_win_arm64_headers)
add_custom_target(nodejs_binding_wrapper ALL
COMMAND ${NPM_CLI} ci
COMMAND ${CMAKE_COMMAND} -E remove_directory ${JS_NODE_ROOT}/build/
COMMAND ${CMAKE_COMMAND} -E make_directory ${JS_NODE_ROOT}/build/
COMMAND cd ${JS_NODE_ROOT} && ${CMAKE_COMMAND} ${JS_NODE_ROOT} --no-warn-unused-cli -G"Visual Studio 16 2019" -A"ARM64" -DCMAKE_RUNTIME_OUTPUT_DIRECTORY="${JS_NODE_ROOT}/build/" -DCMAKE_JS_INC="${nodejs_win_arm64_headers_SOURCE_DIR}/include/node" -DCMAKE_JS_SRC="${JS_NODE_ROOT}/node_modules/cmake-js/lib/cpp/win_delay_load_hook.cc" -DNODE_RUNTIME="node" -DNODE_RUNTIMEVERSION="16.2.0" -DNODE_ARCH="arm64" -DCMAKE_JS_LIB="${nodejs_win_arm64_node_lib_SOURCE_DIR}/node.lib" -Dnapi_build_version="3" -DCMAKE_BUILD_TYPE="RelWithDebInfo" -DONNXRUNTIME_BUILD_DIR="${CMAKE_CURRENT_BINARY_DIR}" -DCMAKE_SHARED_LINKER_FLAGS="/DELAYLOAD:NODE.EXE" -B ./build
COMMAND ${CMAKE_COMMAND} --build ${JS_NODE_ROOT}/build/ --config RelWithDebInfo
WORKING_DIRECTORY ${JS_NODE_ROOT}
COMMENT "Using custom script to build OnnxRuntime Node.js binding")
else()
add_custom_target(nodejs_binding_wrapper ALL
COMMAND ${NPM_CLI} ci
COMMAND ${NPM_CLI} run build -- --onnxruntime-build-dir=${CMAKE_CURRENT_BINARY_DIR} --config=${CMAKE_BUILD_TYPE} --arch=${NODEJS_BINDING_ARCH}
WORKING_DIRECTORY ${JS_NODE_ROOT}
COMMENT "Using cmake-js to build OnnxRuntime Node.js binding")
endif()
add_dependencies(js_common_npm_ci js_npm_ci)
add_dependencies(nodejs_binding_wrapper js_common_npm_ci)
add_dependencies(nodejs_binding_wrapper onnxruntime)

View file

@ -51,9 +51,15 @@ if (ONNXRUNTIME_BUILD_DIR && typeof ONNXRUNTIME_BUILD_DIR === 'string') {
args.push(`--CDONNXRUNTIME_BUILD_DIR=${ONNXRUNTIME_BUILD_DIR}`);
}
// set cross-compile for arm64 on macOS
if (os.platform() === 'darwin' && ARCH === 'arm64') {
args.push('--CDCMAKE_OSX_ARCHITECTURES=arm64');
// set CMAKE_OSX_ARCHITECTURES for macOS build
if (os.platform() === 'darwin') {
if (ARCH === 'x64') {
args.push('--CDCMAKE_OSX_ARCHITECTURES=x86_64');
} else if (ARCH === 'arm64') {
args.push('--CDCMAKE_OSX_ARCHITECTURES=arm64');
} else {
throw new Error(`architecture not supported for macOS build: ${ARCH}`);
}
}
// launch cmake-js configure

View file

@ -21,5 +21,5 @@ describe('E2E Tests - InferenceSession.run()', async () => {
const result = await session!.run({'data_0': input0}, ['softmaxout_1']);
assertTensorEqual(result.softmaxout_1, expectedOutput0);
}
}).timeout('120s');
}).timeout(process.arch === 'x64' ? '120s' : 0);
});

View file

@ -0,0 +1,38 @@
steps:
- checkout: none
- task: DownloadPipelineArtifact@0
displayName: 'Download NPM packages'
inputs:
artifactName: NPM_packages
targetPath: '$(Build.BinariesDirectory)/nodejs-artifact'
- script: mkdir e2e_test
workingDirectory: '$(Build.BinariesDirectory)'
- task: PowerShell@2
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
displayName: 'Extract package file name (Windows)'
inputs:
targetType: 'inline'
script: |
Write-Host "##vso[task.setvariable variable=NpmPackageFilesForTest;]$(ls $(Build.BinariesDirectory)/nodejs-artifact/*.tgz)"
workingDirectory: '$(Build.BinariesDirectory)/e2e_test'
- task: CmdLine@2
condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
displayName: 'Extract package file name (POSIX)'
inputs:
script: |
echo "##vso[task.setvariable variable=NpmPackageFilesForTest;]`ls $(Build.BinariesDirectory)/nodejs-artifact/*.tgz`"
workingDirectory: '$(Build.BinariesDirectory)/e2e_test'
- script: |
npm init -y
npm install $(NpmPackageFilesForTest)
node -p "require('onnxruntime-node')"
workingDirectory: '$(Build.BinariesDirectory)/e2e_test'
- template: ../../templates/component-governance-component-detection-steps.yml
parameters :
condition : 'always'
- template: ../../templates/clean-agent-build-directory-step.yml

View file

@ -0,0 +1,19 @@
parameters:
AgentPool: 'Linux-CPU'
JobSuffix: ''
jobs:
- job: Nodejs_Test_${{ parameters.JobSuffix }}
workspace:
clean: all
timeoutInMinutes: 120
pool: ${{ parameters.AgentPool }}
dependsOn:
- Nodejs_Packaging_CPU
condition: succeeded()
variables:
- name: OnnxRuntimeBuildDirectory
value: '$(Build.BinariesDirectory)'
steps:
- template: test.yml

View file

@ -0,0 +1,18 @@
parameters:
JobSuffix: ''
jobs:
- job: Nodejs_Test_${{ parameters.JobSuffix }}
workspace:
clean: all
timeoutInMinutes: 120
pool:
vmImage: 'macOS-11'
dependsOn:
- Nodejs_Packaging_CPU
condition: succeeded()
variables:
- name: OnnxRuntimeBuildDirectory
value: '$(Build.BinariesDirectory)'
steps:
- template: test.yml

View file

@ -0,0 +1,18 @@
parameters:
AgentPool: 'Win-CPU'
JobSuffix: ''
jobs:
- job: Nodejs_Test_${{ parameters.JobSuffix }}
workspace:
clean: all
timeoutInMinutes: 120
pool: ${{ parameters.AgentPool }}
dependsOn:
- Nodejs_Packaging_CPU
condition: succeeded()
variables:
- name: OnnxRuntimeBuildDirectory
value: '$(Build.BinariesDirectory)'
steps:
- template: test.yml

View file

@ -55,11 +55,13 @@ jobs:
ONNXRUNTIME_ARCH: x64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
NODEJS_BINDING_ARCH: x64
ARCH_AARCH64:
BASE_IMAGE: arm64v8/centos:7
ONNXRUNTIME_ARCH: aarch64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
NODEJS_BINDING_ARCH: arm64
pool: 'Linux-CPU'
steps:
- template: set-version-number-variables-step.yml
@ -76,7 +78,7 @@ jobs:
mkdir -p $HOME/.onnx
docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)"
workingDirectory: $(Build.SourcesDirectory)
@ -103,6 +105,12 @@ jobs:
artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: '$(NODEJS_BINDING_ARCH)'
os: 'linux'
artifactName: 'drop-onnxruntime-nodejs-linux-$(ONNXRUNTIME_ARCH)'
- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
@ -140,18 +148,24 @@ jobs:
- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
MacosArch: arm64
BuildJava: true
BuildNodejs: true
- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
MacosArch: universal2
BuildJava: false
BuildNodejs: false
- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --build_nodejs
MacosArch: x86_64
BuildJava: true
BuildNodejs: true
- task: PublishPipelineArtifact@1
inputs:
@ -238,6 +252,7 @@ jobs:
buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
runTests: ${{ parameters.RunOnnxRuntimeTests }}
buildJava: false
buildNodejs: false
- template: win-ci.yml
parameters:
@ -251,6 +266,7 @@ jobs:
buildparameter: --arm ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
runTests: false
buildJava: false
buildNodejs: false
- template: win-ci.yml
parameters:
@ -261,9 +277,10 @@ jobs:
buildArch: x64
msbuildPlatform: arm64
packageName: arm64
buildparameter: --arm64 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
buildparameter: --build_nodejs --arm64 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
runTests: false
buildJava: false
buildNodejs: true
- template: win-ci.yml
parameters:
@ -274,10 +291,10 @@ jobs:
buildArch: x64
msbuildPlatform: x64
packageName: x64
buildparameter: --build_java ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
buildparameter: --build_java --build_nodejs ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
runTests: ${{ parameters.RunOnnxRuntimeTests }}
buildJava: true
buildNodejs: true
- job: Jar_Packaging
workspace:
@ -543,6 +560,241 @@ jobs:
displayName: 'Clean Agent Directories'
condition: always()
- job: Nodejs_Packaging_CPU
workspace:
clean: all
pool: 'Win-CPU-2021'
variables:
${{ if eq(parameters.IsReleaseBuild, true) }}:
NpmPackagingMode: 'release'
${{ if not(eq(parameters.IsReleaseBuild, true)) }}:
NpmPackagingMode: 'dev'
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
dependsOn:
- Linux_C_API_Packaging_CPU_x64
- MacOS_C_API_Packaging_CPU_x64
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}
condition: succeeded()
steps:
- checkout: self
submodules: true
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet (Win x64)'
inputs:
artifactName: 'onnxruntime-win-x64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet (Win ARM64)'
inputs:
artifactName: 'onnxruntime-win-arm64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet (OSX)'
inputs:
artifactName: 'onnxruntime-osx'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet (Linux x64)'
inputs:
artifactName: 'onnxruntime-linux-x64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet (Linux aarch64)'
inputs:
artifactName: 'onnxruntime-linux-aarch64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (Win x64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-win-x64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/win32/x64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (Win ARM64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-win-arm64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/win32/arm64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (macOS x86_64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-osx-x86_64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/darwin/x64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (macOS arm64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-osx-arm64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/darwin/arm64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (Linux x64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-linux-x64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/linux/x64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - Nodejs (Linux aarch64)'
inputs:
artifactName: 'drop-onnxruntime-nodejs-linux-aarch64'
targetPath: '$(Build.BinariesDirectory)/nodejs-artifacts/linux/arm64/'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet'
inputs:
artifactName: 'drop-extra'
targetPath: '$(Build.BinariesDirectory)/extra-artifact'
- task: PowerShell@2
displayName: 'PowerShell Script'
inputs:
targetType: filePath
filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1
- script: |
dir
workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact'
displayName: 'List artifacts'
- script: |
npm ci
workingDirectory: '$(Build.SourcesDirectory)/js'
displayName: 'Install NPM packages /js'
- script: |
npm ci
workingDirectory: '$(Build.SourcesDirectory)/js/common'
displayName: 'Install NPM packages /js/common'
- script: |
npm ci
workingDirectory: '$(Build.SourcesDirectory)/js/node'
displayName: 'Install NPM packages /js/node'
# Node.js binding win32/x64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64\lib'
Contents: '*.dll'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\x64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\win32\x64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\x64'
# Node.js binding win32/arm64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-arm64\lib'
Contents: '*.dll'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\arm64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\win32\arm64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\arm64'
# Node.js binding linux/x64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64\lib'
Contents: 'libonnxruntime.so.*'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\linux\x64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\x64'
# Node.js binding linux/arm64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-aarch64\lib'
Contents: 'libonnxruntime.so.*'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\arm64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\linux\arm64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\linux\arm64'
# Node.js binding darwin/x64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-osx-x86_64\lib'
Contents: 'libonnxruntime.*.dylib'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\x64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\x64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\darwin\x64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\x64'
# Node.js binding darwin/arm64
- task: CopyFiles@2
displayName: 'Copy nuget binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-osx-arm64\lib'
Contents: 'libonnxruntime.*.dylib'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\arm64'
- task: CopyFiles@2
displayName: 'Copy nodejs binaries to: $(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\arm64\'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\nodejs-artifacts\darwin\arm64'
Contents: '*.node'
TargetFolder: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\darwin\arm64'
- task: PowerShell@2
inputs:
filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\js\pack-npm-packages.ps1'
arguments: '$(NpmPackagingMode) $(Build.SourcesDirectory) node'
workingDirectory: $(Build.BinariesDirectory)
errorActionPreference: stop
displayName: 'Pack NPM packages'
- task: CopyFiles@2
displayName: 'Copy onnxruntime-common package to staging directory'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/js/common/'
Contents: 'onnxruntime-common-*.tgz'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: CopyFiles@2
displayName: 'Copy onnxruntime-node package to staging directory'
inputs:
SourceFolder: '$(Build.SourcesDirectory)/js/node/'
Contents: 'onnxruntime-node-*.tgz'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishPipelineArtifact@0
displayName: 'Publish NPM packages files'
inputs:
artifactName: 'NPM_packages'
targetPath: '$(Build.ArtifactStagingDirectory)'
- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- template: ../nuget/templates/test_win.yml
parameters:
AgentPool : 'Win-CPU-2021'
@ -561,6 +813,20 @@ jobs:
AgentPool : macOS-11
ArtifactSuffix: 'CPU'
- template: ../nodejs/templates/test_win.yml
parameters:
AgentPool : 'Win-CPU-2021'
JobSuffix : 'Win_CPU_x64'
- template: ../nodejs/templates/test_linux.yml
parameters:
AgentPool : 'Linux-CPU'
JobSuffix : 'Linux_CPU_x64'
- template: ../nodejs/templates/test_macos.yml
parameters:
JobSuffix : 'macOS_CPU_x64'
- job: Publish_NuGet_Package_And_Report
workspace:
clean: all

View file

@ -12,6 +12,10 @@ parameters:
type: boolean
default: true
- name: BuildNodejs
type: boolean
default: false
steps:
- script: |
rm -rf $(Build.BinariesDirectory)/Release
@ -41,7 +45,7 @@ steps:
displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory'
condition: and(succeeded(), eq('${{ parameters.MacosArch }}', 'x86_64'))
- ${{ if and(ne(parameters.MacosArch, 'universal2'), eq(parameters.buildJava, true)) }}:
- ${{ if eq(parameters.BuildJava, true) }}:
- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'osx-${{ parameters.MacosArch }}'
@ -49,4 +53,14 @@ steps:
artifactName: 'onnxruntime-java-osx-${{ parameters.MacosArch }}'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.dylib'
nativeLibraryName: 'libonnxruntime4j_jni.dylib'
nativeLibraryName: 'libonnxruntime4j_jni.dylib'
- ${{ if eq(parameters.BuildNodejs, true) }}:
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
parameters:
${{ if eq(parameters.MacosArch, 'x86_64') }}:
arch: x64
${{ if eq(parameters.MacosArch, 'arm64') }}:
arch: arm64
os: 'darwin'
artifactName: 'drop-onnxruntime-nodejs-osx-${{ parameters.MacosArch }}'

View file

@ -0,0 +1,16 @@
parameters:
- name: arch
type: string
- name: os
type: string
- name: artifactName
type: string
default: ''
steps:
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.SourcesDirectory)/js/node/bin/napi-v3/${{ parameters.os }}/${{ parameters.arch }}/onnxruntime_binding.node'
artifactName: '${{parameters.artifactName}}'

View file

@ -0,0 +1,27 @@
# sets up common build tools for the windows build machines before build
parameters:
- name: arch
type: string
- name: DoEsrp
displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
type: boolean
default: true
- name: artifactName
type: string
default: ''
steps:
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\${{ parameters.arch }}'
DisplayName: 'ESRP - Sign Node.js binding binaries'
DoEsrp: ${{parameters.DoEsrp}}
Pattern: '*.node'
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\${{ parameters.arch }}\onnxruntime_binding.node'
artifactName: '${{parameters.artifactName}}'

View file

@ -10,15 +10,12 @@ parameters:
default: false
- name: EnvSetupScript
displayName: Package name for nuget
type: string
- name: buildArch
displayName: Package name for nuget
type: string
- name: msbuildPlatform
displayName: Package name for nuget
type: string
- name: packageName
@ -26,7 +23,7 @@ parameters:
type: string
- name: buildparameter
displayName: Package name for nuget
displayName: Additional build commandline parameter
type: string
- name: runTests
@ -37,13 +34,16 @@ parameters:
type: boolean
default: true
- name: buildNodejs
type: boolean
default: false
- name: job_name_suffix
displayName: job name for nuget
type: string
default: ''
- name: ort_build_pool_name
displayName: job name for nuget
type: string
default: 'Win-CPU-2021'
@ -58,6 +58,7 @@ jobs:
clean: all
pool: ${{ parameters.ort_build_pool_name }}
timeoutInMinutes: 300
steps:
- template: telemetry-steps.yml
@ -67,6 +68,11 @@ jobs:
addToPath: true
architecture: ${{ parameters.buildArch }}
- task: NodeTool@0
condition: and(succeeded(), eq('${{ parameters.buildNodejs}}', true))
inputs:
versionSpec: '14.x'
- task: BatchScript@1
displayName: 'setup env'
inputs:
@ -94,7 +100,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
@ -114,7 +120,7 @@ jobs:
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
workingDirectory: '$(Build.BinariesDirectory)'
- script: |
@ -131,6 +137,13 @@ jobs:
commitId: $(OnnxRuntimeGitCommitHash)
DoEsrp: ${{ parameters.DoEsrp }}
- ${{ if eq(parameters.buildNodejs, true) }}:
- template: nodejs-artifacts-package-and-publish-steps-windows.yml
parameters:
arch: ${{ parameters.packageName }}
artifactName: 'drop-onnxruntime-nodejs-win-${{ parameters.packageName }}'
DoEsrp: ${{ parameters.DoEsrp }}
#Upload protoc.exe, which will be used in nuget build for generating C# files
- task: PublishPipelineArtifact@1
condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64'))
@ -245,7 +258,7 @@ jobs:
condition: and (succeeded(), eq(variables['msbuildPlatform'], 'x64'))
inputs:
msBuildArchitecture: amd64
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)'

View file

@ -0,0 +1,138 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
# This script makes NPM packages for onnxruntime-common and onnxruntime-(node|web)
#
# Release Mode:
# Do not update version number. Version number should be matching $(ORT_ROOT)/VERSION_NUMBER
# Always generate packages for onnxruntime-common and onnxruntime-(node|web)
#
# Dev Mode:
# Compare current content with latest @dev package for onnxruntime-common. If no change, we
# don't publish a new version of onnxruntime-common. Instead, we update the dependency version
# to use the existing version.
#
if ($Args.Count -ne 3) {
throw "This script requires 3 arguments, but got $($Args.Count)"
}
$MODE=$Args[0] # "dev" or "release"
$ORT_ROOT=$Args[1] # eg. D:\source\onnxruntime
$TARGET=$Args[2] # "node" or "web"
$JS_COMMON_DIR="$ORT_ROOT\js\common"
$JS_TARGET_DIR="$ORT_ROOT\js\$TARGET"
if ($MODE -eq "dev") {
# For @dev builds, we compares the following 2 package versions for onnxruntime-common:
# - 'current': the version we are currently building.
# - 'latest': the latest @dev version from npm.js repository.
#
# If the contents of the 2 versions are identical, we don't publish a new version. Instead,
# we only publish onnxruntime-node/onnxruntime-web and set its dependency's version to the
# 'latest'.
# check latest @dev version
Write-Host "Start checking version for onnxruntime-common@dev"
npm view onnxruntime-common@dev --json | Out-File -Encoding utf8 ./ort_common_latest_info.json
$ort_common_latest_version=node -p "require('./ort_common_latest_info.json').version"
$ort_common_latest_dist_tarball=node -p "require('./ort_common_latest_info.json').dist.tarball"
$ort_common_latest_dist_shasum=node -p "require('./ort_common_latest_info.json').dist.shasum"
Write-Host "latest version is: $ort_common_latest_version"
# download package latest@dev
Invoke-WebRequest $ort_common_latest_dist_tarball -OutFile ./latest.tgz
if ($(Get-FileHash -Algorithm SHA1 .\latest.tgz).Hash -ne "$ort_common_latest_dist_shasum") {
throw "SHASUM mismatch"
}
Write-Host "Tarball downloaded"
# generate @dev version
pushd $ORT_ROOT
$dev_version_base=Get-Content .\VERSION_NUMBER
$dev_version_date=$(Get-Date).ToUniversalTime().ToString("yyyyMMdd")
$dev_version_commit=git rev-parse HEAD
$dev_version_commit_short=git rev-parse --short HEAD
popd
$dev_version_number="$dev_version_base-dev.$dev_version_date-$dev_version_commit_short"
Write-Host "Generated @dev current version: $dev_version_number"
# make package for latest
pushd $JS_COMMON_DIR
npm version $ort_common_latest_version
echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
npm pack
popd
$current_tgz_compare_only="$JS_COMMON_DIR\onnxruntime-common-$ort_common_latest_version.tgz"
if (!(Test-Path $current_tgz_compare_only)) {
throw "File is not generated: $current_tgz_compare_only"
}
Write-Host "Package file ready for comparing: $current_tgz_compare_only"
# extract packages
mkdir ./latest
pushd ./latest
tar -xzf ../latest.tgz
rm ../latest.tgz
popd
mkdir ./current
pushd ./current
tar -xzf $current_tgz_compare_only
rm $current_tgz_compare_only
popd
# setup temp folder to compare
mkdir ./temp
pushd ./temp
npm init -y
npm install "dir-compare-cli@1.0.1" "json-diff@0.5.4"
Write-Host "Compare package.json"
npx json-diff ..\latest\package\package.json ..\current\package\package.json
$use_latest=$?
Write-Host "Result: $use_latest"
if ($use_latest) {
# package.json matches. now check package contents.
# do not compare commit number
if (test-path ../latest/package/__commit.txt) { rm ../latest/package/__commit.txt }
if (test-path ../current/package/__commit.txt) { rm ../current/package/__commit.txt }
# skip package.json, we already checked them
rm ../latest/package/package.json
rm ../current/package/package.json
Write-Host "Compare package contents"
npx dircompare -c ../latest/package/ ../current/package/
$use_latest=$?
Write-Host "Result: $use_latest"
}
popd
if (!$use_latest) {
Write-Host "Need update to onnxruntime-common@dev"
# need to publish a new version for onnxruntime-common
pushd $JS_COMMON_DIR
npm version $dev_version_number
# file __commit.txt is already generated
npm pack
popd
}
# make package for target
pushd $JS_TARGET_DIR
npm version $dev_version_number
echo $dev_version_commit | Out-File -Encoding ascii -NoNewline -FilePath ./__commit.txt
npm pack
popd
} else {
# release mode. always publish new package
pushd $JS_COMMON_DIR
npm pack
popd
pushd $JS_TARGET_DIR
npm pack
popd
}

View file

@ -63,9 +63,18 @@ cd ninja-1.10.0
cmake -Bbuild-cmake -H.
cmake --build build-cmake
mv ./build-cmake/ninja /usr/bin
echo "Installing Node.js"
GetFile https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-x64.tar.gz /tmp/src/node-v12.16.3-linux-x64.tar.gz
tar --strip 1 -xf /tmp/src/node-v12.16.3-linux-x64.tar.gz -C /usr
CPU_ARCH=`uname -m`
if [[ "$CPU_ARCH" = "x86_64" ]]; then
NODEJS_ARCH=x64
elif [[ "$CPU_ARCH" = "aarch64" ]]; then
NODEJS_ARCH=arm64
else
NODEJS_ARCH=$CPU_ARCH
fi
GetFile https://nodejs.org/dist/v14.18.1/node-v14.18.1-linux-${NODEJS_ARCH}.tar.gz /tmp/src/node-v14.18.1-linux-${NODEJS_ARCH}.tar.gz
tar --strip 1 -xf /tmp/src/node-v14.18.1-linux-${NODEJS_ARCH}.tar.gz -C /usr
cd /tmp/src
GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip