From 18dc0ec39f867985b15ebe89b8e5f0ced90c4be0 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 12 May 2020 15:31:20 -0700 Subject: [PATCH] Rework jar by creating os-arch folders (#3849) Detect os and arch and move the artifacts to a new folder. Remove unnecesary jars so we cam focus on those we publish. Add signing Make signature simlper. Fix indent. Halt on 32-bit arch. Credits: @Craigacp --- cmake/onnxruntime_java.cmake | 63 ++++++++++++++----- java/build.gradle | 63 ++++++++++++------- .../main/java/ai/onnxruntime/OnnxRuntime.java | 30 ++++++++- 3 files changed, 116 insertions(+), 40 deletions(-) diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake index 6ef9b5baff..0c4338fc93 100644 --- a/cmake/onnxruntime_java.cmake +++ b/cmake/onnxruntime_java.cmake @@ -96,37 +96,68 @@ target_link_libraries(onnxruntime4j_jni PUBLIC onnxruntime) set(JAVA_PACKAGE_OUTPUT_DIR ${JAVA_OUTPUT_DIR}/build) file(MAKE_DIRECTORY ${JAVA_PACKAGE_OUTPUT_DIR}) if (CMAKE_SYSTEM_NAME STREQUAL "Android") - set(ANDROID_PACKAGE_OUTPUT_DIR ${JAVA_PACKAGE_OUTPUT_DIR}/android) - file(MAKE_DIRECTORY ${ANDROID_PACKAGE_OUTPUT_DIR}) + set(ANDROID_PACKAGE_OUTPUT_DIR ${JAVA_PACKAGE_OUTPUT_DIR}/android) + file(MAKE_DIRECTORY ${ANDROID_PACKAGE_OUTPUT_DIR}) endif() + +# Set platform and ach for packaging +if(CMAKE_SIZEOF_VOID_P EQUAL "8") + set(JNI_ARCH x64) +else() + message(FATAL_ERROR "Java is currently not supported for x86 architecture") +endif() + +if (WIN32) + set(JAVA_PLAT "win") +elseif (APPLE) + set(JAVA_PLAT "osx") +elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(JAVA_PLAT "linux") +else() + # We don't do distribution for Android + # Set for completeness + set(JAVA_PLAT "android") + endif() + +# Similar to Nuget schema +set(JAVA_OS_ARCH ${JAVA_PLAT}-${JNI_ARCH}) + # expose native libraries to the gradle build process -set(JAVA_PACKAGE_DIR ai/onnxruntime/native/) +set(JAVA_PACKAGE_DIR ai/onnxruntime/native/${JAVA_OS_ARCH}) set(JAVA_NATIVE_LIB_DIR ${JAVA_OUTPUT_DIR}/native-lib) set(JAVA_NATIVE_JNI_DIR ${JAVA_OUTPUT_DIR}/native-jni) set(JAVA_PACKAGE_LIB_DIR ${JAVA_NATIVE_LIB_DIR}/${JAVA_PACKAGE_DIR}) set(JAVA_PACKAGE_JNI_DIR ${JAVA_NATIVE_JNI_DIR}/${JAVA_PACKAGE_DIR}) file(MAKE_DIRECTORY ${JAVA_PACKAGE_LIB_DIR}) file(MAKE_DIRECTORY ${JAVA_PACKAGE_JNI_DIR}) + if (CMAKE_SYSTEM_NAME STREQUAL "Android") - set(ANDROID_PACKAGE_JNILIBS_DIR ${JAVA_OUTPUT_DIR}/android) - set(ANDROID_PACKAGE_ABI_DIR ${ANDROID_PACKAGE_JNILIBS_DIR}/${ANDROID_ABI}) - file(MAKE_DIRECTORY ${ANDROID_PACKAGE_JNILIBS_DIR}) - file(MAKE_DIRECTORY ${ANDROID_PACKAGE_ABI_DIR}) + set(ANDROID_PACKAGE_JNILIBS_DIR ${JAVA_OUTPUT_DIR}/android) + set(ANDROID_PACKAGE_ABI_DIR ${ANDROID_PACKAGE_JNILIBS_DIR}/${ANDROID_ABI}) + file(MAKE_DIRECTORY ${ANDROID_PACKAGE_JNILIBS_DIR}) + file(MAKE_DIRECTORY ${ANDROID_PACKAGE_ABI_DIR}) endif() + # On Windows TARGET_LINKER_FILE_NAME is the .lib, TARGET_FILE_NAME is the .dll if (WIN32) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) else() - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_LIB_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_PACKAGE_JNI_DIR}/$) endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Android") - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${ANDROID_PACKAGE_ABI_DIR}/$) - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${ANDROID_PACKAGE_ABI_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${ANDROID_PACKAGE_ABI_DIR}/$) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${ANDROID_PACKAGE_ABI_DIR}/$) endif() + # run the build process (this copies the results back into CMAKE_CURRENT_BINARY_DIR) -add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} cmakeBuild -DcmakeBuildDir=${CMAKE_CURRENT_BINARY_DIR} WORKING_DIRECTORY ${JAVA_ROOT}) -if (CMAKE_SYSTEM_NAME STREQUAL "Android") - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} -b build-android.gradle -c settings-android.gradle build -DjniLibsDir=${ANDROID_PACKAGE_JNILIBS_DIR} -DbuildDir=${ANDROID_PACKAGE_OUTPUT_DIR} WORKING_DIRECTORY ${JAVA_ROOT}) +set(GRADLE_ARGS cmakeBuild -DcmakeBuildDir=${CMAKE_CURRENT_BINARY_DIR}) +if(WIN32) + set(GRADLE_ARGS ${GRADLE_ARGS} -Dorg.gradle.daemon=false) +endif() +add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} ${GRADLE_ARGS} WORKING_DIRECTORY ${JAVA_ROOT}) +if (CMAKE_SYSTEM_NAME STREQUAL "Android") + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} -b build-android.gradle -c settings-android.gradle build -DjniLibsDir=${ANDROID_PACKAGE_JNILIBS_DIR} -DbuildDir=${ANDROID_PACKAGE_OUTPUT_DIR} WORKING_DIRECTORY ${JAVA_ROOT}) endif() diff --git a/java/build.gradle b/java/build.gradle index 1119a2c7d7..9dd6861e6f 100644 --- a/java/build.gradle +++ b/java/build.gradle @@ -1,7 +1,8 @@ plugins { - id 'java' - id 'jacoco' + id 'java-library' id 'maven-publish' + id 'signing' + id 'jacoco' id 'com.diffplug.gradle.spotless' version '3.26.0' } @@ -11,19 +12,17 @@ allprojects { } } +project.group = "ai.onnxruntime" +version = rootProject.file('../VERSION_NUMBER').text.trim() + java { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } -project.group = "ai.onnxruntime" -version = rootProject.file('../VERSION_NUMBER').text.trim() - +// This jar tasks serves as a CMAKE signalling +// mechanism. The jar will be overwritten by allJar task jar { - into("META-INF/maven/$project.group/$project.name") { - from { generatePomFileForMavenPublication } - rename ".*", "pom.xml" - } } // Add explicit sources jar with pom file. @@ -70,6 +69,9 @@ def cmakeNativeJniDir = "${cmakeJavaDir}/native-jni" def cmakeNativeTestDir = "${cmakeJavaDir}/native-test" def cmakeBuildOutputDir = "${cmakeJavaDir}/build" +def mavenUser = System.properties['mavenUser'] +def mavenPwd = System.properties['mavenPwd'] + compileJava { options.compilerArgs += ["-h", "${project.buildDir}/headers/"] } @@ -94,22 +96,12 @@ sourceSets.test { if (cmakeBuildDir != null) { // generate tasks to be called from cmake - task jniJar(type: Jar) { - classifier = 'jni' - from cmakeNativeJniDir - } - - task libJar(type: Jar) { - classifier = 'lib' - from cmakeNativeLibDir - } - + // Overwrite jar location task allJar(type: Jar) { into("META-INF/maven/$project.group/$project.name") { from { generatePomFileForMavenPublication } rename ".*", "pom.xml" } - classifier = 'all' from sourceSets.main.output from cmakeNativeJniDir from cmakeNativeLibDir @@ -121,9 +113,6 @@ if (cmakeBuildDir != null) { include 'docs/**' into cmakeBuildOutputDir } - cmakeBuild.dependsOn jar - cmakeBuild.dependsOn jniJar - cmakeBuild.dependsOn libJar cmakeBuild.dependsOn allJar cmakeBuild.dependsOn sourcesJar cmakeBuild.dependsOn javadocJar @@ -189,7 +178,35 @@ publishing { developerConnection = 'scm:git:ssh://github.com/microsoft/onnxruntime.git' url = 'http://github.com/microsoft/onnxruntime' } + developers { + developer { + id = 'onnxruntime' + name = 'ONNX Runtime' + email = 'onnxruntime@microsoft.com' + } + } + } + } + } + repositories { + maven { + url 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' + credentials { + username mavenUser + password mavenPwd } } } } + +// Generates a task signMavenPublication that will +// build all artifacts. +signing { + // Queries env vars: + // ORG_GRADLE_PROJECT_signingKey + // ORG_GRADLE_PROJECT_signingPassword but can be changed to properties + def signingKey = findProperty("signingKey") + def signingPassword = findProperty("signingPassword") + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.maven +} diff --git a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java index 12a8c42881..88aaa17ab9 100644 --- a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java +++ b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,6 +33,8 @@ final class OnnxRuntime { /** The short name of the ONNX runtime JNI shared library */ static final String ONNXRUNTIME_JNI_LIBRARY_NAME = "onnxruntime4j_jni"; + private static final String OS_ARCH_STR = initOsArch(); + private static boolean loaded = false; /** The API handle. */ @@ -39,6 +42,31 @@ final class OnnxRuntime { private OnnxRuntime() {} + /* Computes and initializes OS_ARCH_STR (such as linux-x64) */ + private static String initOsArch() { + String detectedOS = null; + String os = System.getProperty("os.name", "generic").toLowerCase(Locale.ENGLISH); + if (os.contains("mac") || os.contains("darwin")) { + detectedOS = "osx"; + } else if (os.contains("win")) { + detectedOS = "win"; + } else if (os.contains("nux")) { + detectedOS = "linux"; + } else { + detectedOS = "android"; + } + String detectedArch = null; + String arch = System.getProperty("os.arch", "generic").toLowerCase(Locale.ENGLISH); + if (arch.indexOf("amd64") == 0) { + detectedArch = "x64"; + } else if (arch.indexOf("x86") == 0) { + detectedArch = "x86"; + } else { + throw new IllegalStateException("Unsupported arch:" + arch); + } + return detectedOS + '-' + detectedArch; + } + /** * Loads the native C library. * @@ -136,7 +164,7 @@ final class OnnxRuntime { // generate a platform specific library name // replace Mac's jnilib extension to dylib String libraryFileName = System.mapLibraryName(library).replace("jnilib", "dylib"); - String resourcePath = "/ai/onnxruntime/native/" + libraryFileName; + String resourcePath = "/ai/onnxruntime/native/" + OS_ARCH_STR + '/' + libraryFileName; File tempFile = tempDirectory.resolve(libraryFileName).toFile(); try (InputStream is = OnnxRuntime.class.getResourceAsStream(resourcePath)) { if (is == null) {