diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index 9016eb1beff..5233b62cef0 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -37,7 +37,6 @@ runs: shell: bash env: BRANCH: ${{ inputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-${{ inputs.arch-for-build-env }}-build" AWS_DEFAULT_REGION: us-east-1 PR_NUMBER: ${{ github.event.pull_request.number }} @@ -51,7 +50,6 @@ runs: export container_name container_name=$(docker run \ -e BUILD_ENVIRONMENT \ - -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e AWS_DEFAULT_REGION \ -e PR_NUMBER \ diff --git a/.github/templates/common.yml.j2 b/.github/templates/common.yml.j2 index 798247fcb49..ca1ec0b84f0 100644 --- a/.github/templates/common.yml.j2 +++ b/.github/templates/common.yml.j2 @@ -91,7 +91,6 @@ on: AWS_DEFAULT_REGION: us-east-1 GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: !{{ build_environment }}-test PR_NUMBER: ${{ github.event.pull_request.number }} SHA1: ${{ github.event.pull_request.head.sha || github.sha }} TAG: ${{ steps.parse-ref.outputs.tag }} diff --git a/.github/workflows/_android-build-test.yml b/.github/workflows/_android-build-test.yml index d79b98eeb2a..f1de65c53a7 100644 --- a/.github/workflows/_android-build-test.yml +++ b/.github/workflows/_android-build-test.yml @@ -57,7 +57,6 @@ jobs: - name: Build env: BUILD_ENVIRONMENT: ${{ inputs.build-environment }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test TORCH_CUDA_ARCH_LIST: 5.2 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} @@ -79,7 +78,6 @@ jobs: git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0 export id id=$(docker run -e BUILD_ENVIRONMENT \ - -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e SKIP_SCCACHE_INITIALIZATION=1 \ diff --git a/.github/workflows/_bazel-build-test.yml b/.github/workflows/_bazel-build-test.yml index 3fb29593260..fae10e3bf72 100644 --- a/.github/workflows/_bazel-build-test.yml +++ b/.github/workflows/_bazel-build-test.yml @@ -68,7 +68,6 @@ jobs: env: BUILD_ENVIRONMENT: ${{ inputs.build-environment }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test # TODO duplicated AWS_DEFAULT_REGION: us-east-1 SHA1: ${{ github.event.pull_request.head.sha || github.sha }} @@ -80,7 +79,6 @@ jobs: # detached container should get cleaned up by teardown_ec2_linux container_name=$(docker run \ -e BUILD_ENVIRONMENT \ - -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e SKIP_SCCACHE_INITIALIZATION=1 \ @@ -104,7 +102,6 @@ jobs: # Time out the test phase after 3.5 hours timeout-minutes: 210 env: - JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test BUILD_ENVIRONMENT: ${{ inputs.build-environment }} PR_NUMBER: ${{ github.event.pull_request.number }} BRANCH: ${{ steps.parse-ref.outputs.branch }} @@ -128,7 +125,6 @@ jobs: -e GIT_DEFAULT_BRANCH="$GIT_DEFAULT_BRANCH" \ -e SHARD_NUMBER \ -e NUM_TEST_SHARDS \ - -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e PYTORCH_RETRY_TEST_CASES \ @@ -163,7 +159,6 @@ jobs: GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test PR_NUMBER: ${{ github.event.pull_request.number }} PYTORCH_RETRY_TEST_CASES: 1 PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1 diff --git a/.github/workflows/_buck-build-test.yml b/.github/workflows/_buck-build-test.yml index ba9761e3254..1d54f55c242 100644 --- a/.github/workflows/_buck-build-test.yml +++ b/.github/workflows/_buck-build-test.yml @@ -10,8 +10,6 @@ defaults: jobs: buck-build-test: runs-on: ubuntu-latest - env: - JOB_BASE_NAME: ubuntu-latest-buck steps: - name: Checkout PyTorch uses: pytorch/pytorch/.github/actions/checkout-pytorch@master diff --git a/.github/workflows/_ios-build-test.yml b/.github/workflows/_ios-build-test.yml index c8098b5a8db..3ec042a4a2b 100644 --- a/.github/workflows/_ios-build-test.yml +++ b/.github/workflows/_ios-build-test.yml @@ -44,7 +44,6 @@ jobs: runs-on: macos-10.15 timeout-minutes: 240 env: - JOB_BASE_NAME: ${{ inputs.build-environment }}-build IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }} IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }} IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }} diff --git a/.github/workflows/_linux-build.yml b/.github/workflows/_linux-build.yml index 9a9cae06f4e..ba4c7aafb0b 100644 --- a/.github/workflows/_linux-build.yml +++ b/.github/workflows/_linux-build.yml @@ -85,7 +85,6 @@ jobs: env: BUILD_ENVIRONMENT: ${{ inputs.build-environment }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-build # TODO duplicated AWS_DEFAULT_REGION: us-east-1 PR_NUMBER: ${{ github.event.pull_request.number }} @@ -102,7 +101,6 @@ jobs: # detached container should get cleaned up by teardown_ec2_linux container_name=$(docker run \ -e BUILD_ENVIRONMENT \ - -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e AWS_DEFAULT_REGION \ -e PR_NUMBER \ diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index ac1ea422952..c3d90b6cbc0 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -75,7 +75,6 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} PYTORCH_RETRY_TEST_CASES: 1 PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1 - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} @@ -115,7 +114,6 @@ jobs: -e AWS_DEFAULT_REGION \ -e IN_WHEEL_TEST \ -e SHARD_NUMBER \ - -e JOB_BASE_NAME \ -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PR_BODY \ @@ -171,7 +169,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} diff --git a/.github/workflows/_mac-build.yml b/.github/workflows/_mac-build.yml index d9747a3ebbd..672e0d31ee4 100644 --- a/.github/workflows/_mac-build.yml +++ b/.github/workflows/_mac-build.yml @@ -40,7 +40,6 @@ jobs: if: github.repository_owner == 'pytorch' runs-on: ${{ inputs.runner-type }} env: - JOB_BASE_NAME: ${{ inputs.build-environment }} # For sccache access (only on non-forked PRs) AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index 651f291ad80..ad874fdccff 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -37,7 +37,6 @@ jobs: env: GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} @@ -100,7 +99,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} diff --git a/.github/workflows/_rocm-test.yml b/.github/workflows/_rocm-test.yml index 61ecf9937f3..004640385e5 100644 --- a/.github/workflows/_rocm-test.yml +++ b/.github/workflows/_rocm-test.yml @@ -73,7 +73,6 @@ jobs: SHA1: ${{ github.event.pull_request.head.sha || github.sha }} PYTORCH_RETRY_TEST_CASES: 1 PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1 - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} @@ -110,7 +109,6 @@ jobs: -e AWS_DEFAULT_REGION \ -e IN_WHEEL_TEST \ -e SHARD_NUMBER \ - -e JOB_BASE_NAME \ -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PR_BODY \ @@ -164,7 +162,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} diff --git a/.github/workflows/_win-build.yml b/.github/workflows/_win-build.yml index 71cad030e73..a8b1edabc4a 100644 --- a/.github/workflows/_win-build.yml +++ b/.github/workflows/_win-build.yml @@ -26,8 +26,6 @@ jobs: if: github.repository_owner == 'pytorch' runs-on: [self-hosted, windows.4xlarge] timeout-minutes: 240 - env: - JOB_BASE_NAME: ${{ inputs.build-environment }}-build steps: # [see note: pytorch repo ref] - name: Checkout PyTorch diff --git a/.github/workflows/_win-test.yml b/.github/workflows/_win-test.yml index f52281ed2cb..ddf8196a58a 100644 --- a/.github/workflows/_win-test.yml +++ b/.github/workflows/_win-test.yml @@ -79,7 +79,6 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} TEST_CONFIG: ${{ matrix.config }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test PR_BODY: ${{ github.event.pull_request.body }} TORCH_CUDA_ARCH_LIST: "7.0" run: | @@ -110,7 +109,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} BRANCH: ${{ steps.parse-ref.outputs.branch }} - JOB_BASE_NAME: ${{ inputs.build-environment }}-test TEST_CONFIG: ${{ matrix.config }} SHARD_NUMBER: ${{ matrix.shard }} BUILD_ENVIRONMENT: ${{ inputs.build-environment }} diff --git a/.github/workflows/run_android_tests.yml b/.github/workflows/run_android_tests.yml index 5b297347563..bfb5c83d103 100644 --- a/.github/workflows/run_android_tests.yml +++ b/.github/workflows/run_android_tests.yml @@ -26,8 +26,6 @@ jobs: build-and-test: runs-on: ubuntu-latest - env: - JOB_BASE_NAME: ubuntu-latest-android-tests steps: - name: Setup miniconda uses: conda-incubator/setup-miniconda@v2 diff --git a/.jenkins/pytorch/build-asan.sh b/.jenkins/pytorch/build-asan.sh index 963a062fa1e..c07c4ddb865 100755 --- a/.jenkins/pytorch/build-asan.sh +++ b/.jenkins/pytorch/build-asan.sh @@ -6,6 +6,8 @@ # shellcheck source=./common.sh source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# shellcheck source=./common-build.sh +source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" echo "Clang version:" clang --version diff --git a/.jenkins/pytorch/build-mobile.sh b/.jenkins/pytorch/build-mobile.sh index 85d8280597e..1f253ff58c0 100755 --- a/.jenkins/pytorch/build-mobile.sh +++ b/.jenkins/pytorch/build-mobile.sh @@ -8,6 +8,8 @@ set -eu -o pipefail # shellcheck source=./common.sh source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# shellcheck source=./common-build.sh +source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" # Install torch & torchvision - used to download & trace test model. # Ideally we should use the libtorch built on the PR so that backward diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh index f4513e2918c..a754af59664 100755 --- a/.jenkins/pytorch/build.sh +++ b/.jenkins/pytorch/build.sh @@ -8,6 +8,8 @@ set -ex # shellcheck source=./common.sh source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +# shellcheck source=./common-build.sh +source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" if [[ "$BUILD_ENVIRONMENT" == *-clang7-asan* ]]; then exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@" diff --git a/.jenkins/pytorch/common-build.sh b/.jenkins/pytorch/common-build.sh new file mode 100644 index 00000000000..4f21d9c6787 --- /dev/null +++ b/.jenkins/pytorch/common-build.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Required environment variables: +# $BUILD_ENVIRONMENT (should be set by your Docker image) + +if [[ "$BUILD_ENVIRONMENT" != *win-* ]]; then + # Save the absolute path in case later we chdir (as occurs in the gpu perf test) + script_dir="$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit ; pwd -P )" + + if which sccache > /dev/null; then + # Save sccache logs to file + sccache --stop-server > /dev/null 2>&1 || true + rm -f ~/sccache_error.log || true + if [[ -n "${SKIP_SCCACHE_INITIALIZATION:-}" ]]; then + # sccache --start-server seems to hang forever on self hosted runners for GHA + # so let's just go ahead and skip the --start-server altogether since it seems + # as though sccache still gets used even when the sscache server isn't started + # explicitly + echo "Skipping sccache server initialization, setting environment variables" + export SCCACHE_IDLE_TIMEOUT=1200 + export SCCACHE_ERROR_LOG=~/sccache_error.log + export RUST_LOG=sccache::server=error + elif [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then + SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server + else + # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR: + # https://github.com/pytorch/pytorch/pull/16645 + SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server + fi + + # Report sccache stats for easier debugging + sccache --zero-stats + function sccache_epilogue() { + echo "::group::Sccache Compilation Log" + echo '=================== sccache compilation log ===================' + python "$script_dir/print_sccache_log.py" ~/sccache_error.log 2>/dev/null + echo '=========== If your build fails, please take a look at the log above for possible reasons ===========' + sccache --show-stats + sccache --stop-server || true + echo "::endgroup::" + } + + trap_add sccache_epilogue EXIT + fi + + if which ccache > /dev/null; then + # Report ccache stats for easier debugging + ccache --zero-stats + ccache --show-stats + function ccache_epilogue() { + ccache --show-stats + } + trap_add ccache_epilogue EXIT + fi +fi diff --git a/.jenkins/pytorch/common.sh b/.jenkins/pytorch/common.sh index 5fc2fcb209d..c71acc7e66c 100644 --- a/.jenkins/pytorch/common.sh +++ b/.jenkins/pytorch/common.sh @@ -5,9 +5,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh" set -ex -# Save the SCRIPT_DIR absolute path in case later we chdir (as occurs in the gpu perf test) -SCRIPT_DIR="$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )" - # Required environment variables: # $BUILD_ENVIRONMENT (should be set by your Docker image) @@ -22,56 +19,6 @@ if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then export HSA_FORCE_FINE_GRAIN_PCIE=1 fi -if [[ "$BUILD_ENVIRONMENT" != *win-* ]]; then - if which sccache > /dev/null; then - # Save sccache logs to file - sccache --stop-server > /dev/null 2>&1 || true - rm -f ~/sccache_error.log || true - if [[ -n "${SKIP_SCCACHE_INITIALIZATION:-}" ]]; then - # sccache --start-server seems to hang forever on self hosted runners for GHA - # so let's just go ahead and skip the --start-server altogether since it seems - # as though sccache still gets used even when the sscache server isn't started - # explicitly - echo "Skipping sccache server initialization, setting environment variables" - export SCCACHE_IDLE_TIMEOUT=1200 - export SCCACHE_ERROR_LOG=~/sccache_error.log - export RUST_LOG=sccache::server=error - elif [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then - SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server - else - # increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR: - # https://github.com/pytorch/pytorch/pull/16645 - SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server - fi - - # Report sccache stats for easier debugging - sccache --zero-stats - function sccache_epilogue() { - echo "::group::Sccache Compilation Log" - echo '=================== sccache compilation log ===================' - python "$SCRIPT_DIR/print_sccache_log.py" ~/sccache_error.log 2>/dev/null - echo '=========== If your build fails, please take a look at the log above for possible reasons ===========' - sccache --show-stats - sccache --stop-server || true - echo "::endgroup::" - } - - if [[ "${JOB_BASE_NAME}" == *-build ]]; then - trap_add sccache_epilogue EXIT - fi - fi - - if which ccache > /dev/null; then - # Report ccache stats for easier debugging - ccache --zero-stats - ccache --show-stats - function ccache_epilogue() { - ccache --show-stats - } - trap_add ccache_epilogue EXIT - fi -fi - # TODO: Renable libtorch testing for MacOS, see https://github.com/pytorch/pytorch/issues/62598 # shellcheck disable=SC2034 BUILD_TEST_LIBTORCH=0 diff --git a/.jenkins/pytorch/docs-test.sh b/.jenkins/pytorch/docs-test.sh index 374dae28e32..d57d78bc7f7 100755 --- a/.jenkins/pytorch/docs-test.sh +++ b/.jenkins/pytorch/docs-test.sh @@ -5,6 +5,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/common.sh" echo "Testing pytorch docs" -cd "${SCRIPT_DIR}/../../docs" +cd docs pip_install -r requirements.txt make doctest diff --git a/.jenkins/pytorch/macos-build.sh b/.jenkins/pytorch/macos-build.sh index 7c1854346ef..5105e6d89b1 100755 --- a/.jenkins/pytorch/macos-build.sh +++ b/.jenkins/pytorch/macos-build.sh @@ -3,6 +3,8 @@ # shellcheck disable=SC2034 # shellcheck source=./macos-common.sh source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh" +# shellcheck source=./common-build.sh +source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" # Build PyTorch if [ -z "${CI}" ]; then diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh index c3bd81318bc..38a73222936 100755 --- a/.jenkins/pytorch/test.sh +++ b/.jenkins/pytorch/test.sh @@ -424,6 +424,10 @@ test_torch_function_benchmark() { } build_xla() { + # xla test needs sccache setup. + # shellcheck source=./common-build.sh + source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" + XLA_DIR=xla USE_CACHE=0 clone_pytorch_xla @@ -438,6 +442,10 @@ build_xla() { } test_xla() { + # xla test needs sccache setup. + # shellcheck source=./common-build.sh + source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" + clone_pytorch_xla # shellcheck disable=SC1091 source "./xla/.circleci/common.sh" @@ -488,6 +496,10 @@ test_forward_backward_compatibility() { test_bazel() { set -e + # bazel test needs sccache setup. + # shellcheck source=./common-build.sh + source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh" + get_bazel # Test //c10/... without Google flags and logging libraries. The diff --git a/.jenkins/pytorch/win-build.sh b/.jenkins/pytorch/win-build.sh index 47011e4a107..2de3022d40c 100755 --- a/.jenkins/pytorch/win-build.sh +++ b/.jenkins/pytorch/win-build.sh @@ -12,6 +12,8 @@ fi SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) # shellcheck source=./common.sh source "$SCRIPT_PARENT_DIR/common.sh" +# shellcheck source=./common-build.sh +source "$SCRIPT_PARENT_DIR/common-build.sh" IMAGE_COMMIT_ID=$(git rev-parse HEAD) export IMAGE_COMMIT_ID diff --git a/.jenkins/pytorch/win-test.sh b/.jenkins/pytorch/win-test.sh index 7455ebe8e76..29c4e8052f6 100755 --- a/.jenkins/pytorch/win-test.sh +++ b/.jenkins/pytorch/win-test.sh @@ -65,7 +65,7 @@ run_tests() { done "$SCRIPT_HELPERS_DIR"/test_python_shard.bat - if [[ ( -z "${JOB_BASE_NAME}" || "${JOB_BASE_NAME}" == *-test ) && $NUM_TEST_SHARDS -eq 1 ]]; then + if [[ $NUM_TEST_SHARDS -eq 1 ]]; then "$SCRIPT_HELPERS_DIR"/test_custom_script_ops.bat "$SCRIPT_HELPERS_DIR"/test_custom_backend.bat "$SCRIPT_HELPERS_DIR"/test_libtorch.bat diff --git a/tools/stats/print_test_stats.py b/tools/stats/print_test_stats.py index c6994c1ea68..159ed6ef99a 100755 --- a/tools/stats/print_test_stats.py +++ b/tools/stats/print_test_stats.py @@ -785,7 +785,9 @@ def build_info() -> ReportMetaMeta: os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", "HEAD")) ), "build_branch": os.environ.get("BRANCH", os.environ.get("CIRCLE_BRANCH", "")), - "build_job": os.environ.get("JOB_BASE_NAME", os.environ.get("CIRCLE_JOB", "")), + "build_job": os.environ.get( + "BUILD_ENVIRONMENT", os.environ.get("CIRCLE_JOB", "") + ), "build_workflow_id": os.environ.get( "WORKFLOW_ID", os.environ.get("CIRCLE_WORKFLOW_ID", "") ), @@ -877,7 +879,7 @@ def assemble_s3_object( def send_report_to_s3(head_report: Version2Report) -> None: - job = os.getenv("JOB_BASE_NAME", os.environ.get("CIRCLE_JOB")) + job = os.getenv("BUILD_ENVIRONMENT", os.environ.get("CIRCLE_JOB")) sha1 = os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", "")) now = datetime.datetime.utcnow().isoformat() @@ -929,7 +931,7 @@ def print_regressions(head_report: Report, *, num_prev_commits: int) -> None: else: commits = commits[:-1] - job = os.environ.get("JOB_BASE_NAME", "") + job = os.environ.get("BUILD_ENVIRONMENT", "") objects: Dict[Commit, List[Report]] = defaultdict(list) for commit in commits: diff --git a/tools/testing/test_selections.py b/tools/testing/test_selections.py index 9056054ec73..78e92745ce2 100644 --- a/tools/testing/test_selections.py +++ b/tools/testing/test_selections.py @@ -21,15 +21,7 @@ class JobTimeJSON(TypedDict): def _get_stripped_CI_job() -> str: - """E.g. convert 'pytorch_windows_vs2019_py36_cuda10.1_build' to 'pytorch_windows_vs2019_py36_cuda10.1'.""" - job = os.environ.get("JOB_BASE_NAME", "").rstrip("0123456789") - if job.endswith("_slow_test"): - job = job[: len(job) - len("_slow_test")] - elif job.endswith("_test") or job.endswith("-test"): - job = job[: len(job) - len("_test")] - elif job.endswith("_build") or job.endswith("-build"): - job = job[: len(job) - len("_build")] - return job + return os.environ.get("BUILD_ENVIRONMENT", "") def _get_job_times_json(job_times: Dict[str, float]) -> JobTimeJSON: