Split binary/symbol and then upload ortsrv nightly build to blob storage (#1120)

* Upload ortsrv nightly build to blob storage

* Fix the binary directory

* Temporarily disable binary split

* Split binary in build container

* Update azcopy command

* Update getopts

* Pass blob sas url as string

* Avoid binary split on Windows

* Update build_server logic in build.py

tools/ci_build/build.py

@@ -577,6 +577,19 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para
           run_subprocess([exe,'-x'] + cmd, cwd=cwd)
 
 
+def split_server_binary_and_symbol(build_dir, configs):
+    if is_windows():
+        # TODO: Windows support
+        pass
+    else:
+        for config in configs:
+            if config == 'RelWithDebInfo':
+                config_build_dir = get_config_build_dir(build_dir, config)
+                run_subprocess(['objcopy', '--only-keep-debug', 'onnxruntime_server', 'onnxruntime_server.symbol'], cwd=config_build_dir)
+                run_subprocess(['strip', '--strip-debug', '--strip-unneeded', 'onnxruntime_server'], cwd=config_build_dir)
+                run_subprocess(['objcopy', '--add-gnu-debuglink=onnxruntime_server.symbol', 'onnxruntime_server'], cwd=config_build_dir)
+            
+
 def run_server_tests(build_dir, configs):
     pip_freeze_result = run_subprocess([sys.executable, '-m', 'pip', 'freeze'], capture=True).stdout
     installed_packages = [r.decode().split('==')[0] for r in pip_freeze_result.split()]
@@ -826,11 +839,12 @@ def main():
               if args.use_mkldnn:
                 run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'mkldnn', True, 1)
 
-    if args.build_server and args.enable_server_tests:
-        run_server_tests(build_dir, configs)
-
-    if args.build_server and args.enable_server_model_tests:
-        run_server_model_tests(build_dir, configs)
+    if args.build_server:
+        split_server_binary_and_symbol(build_dir, configs)
+        if args.enable_server_tests:
+            run_server_tests(build_dir, configs)
+        if args.enable_server_model_tests:
+            run_server_model_tests(build_dir, configs)
 
     if args.build:
         if args.build_wheel:

tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml

@@ -33,6 +33,9 @@ jobs:
     - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--config RelWithDebInfo --build_server --use_openmp --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
       displayName: 'Run build script with model tests'
 
+    - script: 'tools/ci_build/github/linux/upload_ortsrv_binaries.sh -a $(Build.BinariesDirectory) -r $(Build.BinariesDirectory)/RelWithDebInfo -i $(Build.BuildNumber) -c $(Build.SourceVersion) -b "$(blob.binary_upload_url)" -p "--config RelWithDebInfo --build_server --use_openmp --use_full_protobuf --enable_server_model_tests --cmake_extra_defines onnxruntime_SERVER_VERSION=$(cat ./VERSION_NUMBER)-$(Build.BuildNumber) onnxruntime_LATEST_COMMIT_ID=$(Build.SourceVersion)"'
+      displayName: 'Upload binary to blob storage'
+
     - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
       displayName: 'Component Detection'
       condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))

tools/ci_build/github/linux/upload_ortsrv_binaries.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e -o -x
+
+while getopts a:r:i:c:p:b: parameter_Option
+do case "${parameter_Option}"
+in
+a) AZCOPY_DIR=${OPTARG};;
+r) BINARY_DIR=${OPTARG};;
+i) BUILD_ID=${OPTARG};;
+c) LAST_COMMIT_ID=${OPTARG};;
+p) BUILD_PARAMETERS=${OPTARG};;
+b) BLOB_SAS_URL=${OPTARG};;
+esac
+done
+
+echo ""
+echo "ad=$AZCOPY_DIR bd=$BINARY_DIR bi=$BUILD_ID lci=$LAST_COMMIT_ID bc=$BUILD_PARAMETERS bsu=$BLOB_SAS_URL"
+
+echo ""
+echo "Creating temp folder $BINARY_DIR/$BUILD_ID ... "
+mkdir $BINARY_DIR/$BUILD_ID
+cp $BINARY_DIR/onnxruntime_server $BINARY_DIR/$BUILD_ID
+cp $BINARY_DIR/onnxruntime_server.symbol $BINARY_DIR/$BUILD_ID
+
+echo "Create build info file ..."
+echo "Build parameters: $BUILD_PARAMETERS" >> $BINARY_DIR/$BUILD_ID/build_info.txt
+echo "Last commit id: $LAST_COMMIT_ID" >> $BINARY_DIR/$BUILD_ID/build_info.txt
+
+echo "Upload the folder to blob storage ..."
+$AZCOPY_DIR/azcopy cp $BINARY_DIR/$BUILD_ID $BLOB_SAS_URL --recursive=true
+
+echo "Done!"