[ci] delete JOB_BASE_NAME (#80046)

`JOB_BASE_NAME` was a holdover from jenkins compatibility. Eventually,
it morphed to be always set to the build enviroment + `-test` or
`-build`, and we used it to detect whether we were in a build or test.

That's sort of pointless, so removing and fixing up the few remaining
use cases.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/80046
Approved by: https://github.com/malfet, https://github.com/janeyx99
This commit is contained in:
Michael Suo 2022-06-23 10:00:30 -07:00 committed by PyTorch MergeBot
parent 8c0796e57f
commit 5029a91f7b
26 changed files with 84 additions and 97 deletions

View file

@ -37,7 +37,6 @@ runs:
shell: bash
env:
BRANCH: ${{ inputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-${{ inputs.arch-for-build-env }}-build"
AWS_DEFAULT_REGION: us-east-1
PR_NUMBER: ${{ github.event.pull_request.number }}
@ -51,7 +50,6 @@ runs:
export container_name
container_name=$(docker run \
-e BUILD_ENVIRONMENT \
-e JOB_BASE_NAME \
-e MAX_JOBS="$(nproc --ignore=2)" \
-e AWS_DEFAULT_REGION \
-e PR_NUMBER \

View file

@ -91,7 +91,6 @@ on:
AWS_DEFAULT_REGION: us-east-1
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: !{{ build_environment }}-test
PR_NUMBER: ${{ github.event.pull_request.number }}
SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
TAG: ${{ steps.parse-ref.outputs.tag }}

View file

@ -57,7 +57,6 @@ jobs:
- name: Build
env:
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test
TORCH_CUDA_ARCH_LIST: 5.2
SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
@ -79,7 +78,6 @@ jobs:
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
export id
id=$(docker run -e BUILD_ENVIRONMENT \
-e JOB_BASE_NAME \
-e MAX_JOBS="$(nproc --ignore=2)" \
-e SCCACHE_BUCKET \
-e SKIP_SCCACHE_INITIALIZATION=1 \

View file

@ -68,7 +68,6 @@ jobs:
env:
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test
# TODO duplicated
AWS_DEFAULT_REGION: us-east-1
SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
@ -80,7 +79,6 @@ jobs:
# detached container should get cleaned up by teardown_ec2_linux
container_name=$(docker run \
-e BUILD_ENVIRONMENT \
-e JOB_BASE_NAME \
-e MAX_JOBS="$(nproc --ignore=2)" \
-e SCCACHE_BUCKET \
-e SKIP_SCCACHE_INITIALIZATION=1 \
@ -104,7 +102,6 @@ jobs:
# Time out the test phase after 3.5 hours
timeout-minutes: 210
env:
JOB_BASE_NAME: ${{ inputs.build-environment }}-build-and-test
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
@ -128,7 +125,6 @@ jobs:
-e GIT_DEFAULT_BRANCH="$GIT_DEFAULT_BRANCH" \
-e SHARD_NUMBER \
-e NUM_TEST_SHARDS \
-e JOB_BASE_NAME \
-e MAX_JOBS="$(nproc --ignore=2)" \
-e SCCACHE_BUCKET \
-e PYTORCH_RETRY_TEST_CASES \
@ -163,7 +159,6 @@ jobs:
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
PR_NUMBER: ${{ github.event.pull_request.number }}
PYTORCH_RETRY_TEST_CASES: 1
PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1

View file

@ -10,8 +10,6 @@ defaults:
jobs:
buck-build-test:
runs-on: ubuntu-latest
env:
JOB_BASE_NAME: ubuntu-latest-buck
steps:
- name: Checkout PyTorch
uses: pytorch/pytorch/.github/actions/checkout-pytorch@master

View file

@ -44,7 +44,6 @@ jobs:
runs-on: macos-10.15
timeout-minutes: 240
env:
JOB_BASE_NAME: ${{ inputs.build-environment }}-build
IOS_CERT_KEY_2022: ${{ secrets.IOS_CERT_KEY_2022 }}
IOS_CERT_SECRET: ${{ secrets.IOS_CERT_SECRET }}
IOS_DEV_TEAM_ID: ${{ secrets.IOS_DEV_TEAM_ID }}

View file

@ -85,7 +85,6 @@ jobs:
env:
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-build
# TODO duplicated
AWS_DEFAULT_REGION: us-east-1
PR_NUMBER: ${{ github.event.pull_request.number }}
@ -102,7 +101,6 @@ jobs:
# detached container should get cleaned up by teardown_ec2_linux
container_name=$(docker run \
-e BUILD_ENVIRONMENT \
-e JOB_BASE_NAME \
-e MAX_JOBS="$(nproc --ignore=2)" \
-e AWS_DEFAULT_REGION \
-e PR_NUMBER \

View file

@ -75,7 +75,6 @@ jobs:
BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }}
PYTORCH_RETRY_TEST_CASES: 1
PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
NUM_TEST_SHARDS: ${{ matrix.num_shards }}
@ -115,7 +114,6 @@ jobs:
-e AWS_DEFAULT_REGION \
-e IN_WHEEL_TEST \
-e SHARD_NUMBER \
-e JOB_BASE_NAME \
-e TEST_CONFIG \
-e NUM_TEST_SHARDS \
-e PR_BODY \
@ -171,7 +169,6 @@ jobs:
AWS_DEFAULT_REGION: us-east-1
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}

View file

@ -40,7 +40,6 @@ jobs:
if: github.repository_owner == 'pytorch'
runs-on: ${{ inputs.runner-type }}
env:
JOB_BASE_NAME: ${{ inputs.build-environment }}
# For sccache access (only on non-forked PRs)
AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}

View file

@ -37,7 +37,6 @@ jobs:
env:
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
NUM_TEST_SHARDS: ${{ matrix.num_shards }}
@ -100,7 +99,6 @@ jobs:
AWS_DEFAULT_REGION: us-east-1
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}

View file

@ -73,7 +73,6 @@ jobs:
SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
PYTORCH_RETRY_TEST_CASES: 1
PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
NUM_TEST_SHARDS: ${{ matrix.num_shards }}
@ -110,7 +109,6 @@ jobs:
-e AWS_DEFAULT_REGION \
-e IN_WHEEL_TEST \
-e SHARD_NUMBER \
-e JOB_BASE_NAME \
-e TEST_CONFIG \
-e NUM_TEST_SHARDS \
-e PR_BODY \
@ -164,7 +162,6 @@ jobs:
AWS_DEFAULT_REGION: us-east-1
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}

View file

@ -26,8 +26,6 @@ jobs:
if: github.repository_owner == 'pytorch'
runs-on: [self-hosted, windows.4xlarge]
timeout-minutes: 240
env:
JOB_BASE_NAME: ${{ inputs.build-environment }}-build
steps:
# [see note: pytorch repo ref]
- name: Checkout PyTorch

View file

@ -79,7 +79,6 @@ jobs:
SHARD_NUMBER: ${{ matrix.shard }}
NUM_TEST_SHARDS: ${{ matrix.num_shards }}
TEST_CONFIG: ${{ matrix.config }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
PR_BODY: ${{ github.event.pull_request.body }}
TORCH_CUDA_ARCH_LIST: "7.0"
run: |
@ -110,7 +109,6 @@ jobs:
AWS_DEFAULT_REGION: us-east-1
GIT_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
BRANCH: ${{ steps.parse-ref.outputs.branch }}
JOB_BASE_NAME: ${{ inputs.build-environment }}-test
TEST_CONFIG: ${{ matrix.config }}
SHARD_NUMBER: ${{ matrix.shard }}
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}

View file

@ -26,8 +26,6 @@ jobs:
build-and-test:
runs-on: ubuntu-latest
env:
JOB_BASE_NAME: ubuntu-latest-android-tests
steps:
- name: Setup miniconda
uses: conda-incubator/setup-miniconda@v2

View file

@ -6,6 +6,8 @@
# shellcheck source=./common.sh
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
echo "Clang version:"
clang --version

View file

@ -8,6 +8,8 @@ set -eu -o pipefail
# shellcheck source=./common.sh
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
# Install torch & torchvision - used to download & trace test model.
# Ideally we should use the libtorch built on the PR so that backward

View file

@ -8,6 +8,8 @@ set -ex
# shellcheck source=./common.sh
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
if [[ "$BUILD_ENVIRONMENT" == *-clang7-asan* ]]; then
exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"

View file

@ -0,0 +1,54 @@
#!/bin/bash
# Required environment variables:
# $BUILD_ENVIRONMENT (should be set by your Docker image)
if [[ "$BUILD_ENVIRONMENT" != *win-* ]]; then
# Save the absolute path in case later we chdir (as occurs in the gpu perf test)
script_dir="$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit ; pwd -P )"
if which sccache > /dev/null; then
# Save sccache logs to file
sccache --stop-server > /dev/null 2>&1 || true
rm -f ~/sccache_error.log || true
if [[ -n "${SKIP_SCCACHE_INITIALIZATION:-}" ]]; then
# sccache --start-server seems to hang forever on self hosted runners for GHA
# so let's just go ahead and skip the --start-server altogether since it seems
# as though sccache still gets used even when the sscache server isn't started
# explicitly
echo "Skipping sccache server initialization, setting environment variables"
export SCCACHE_IDLE_TIMEOUT=1200
export SCCACHE_ERROR_LOG=~/sccache_error.log
export RUST_LOG=sccache::server=error
elif [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
else
# increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
# https://github.com/pytorch/pytorch/pull/16645
SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
fi
# Report sccache stats for easier debugging
sccache --zero-stats
function sccache_epilogue() {
echo "::group::Sccache Compilation Log"
echo '=================== sccache compilation log ==================='
python "$script_dir/print_sccache_log.py" ~/sccache_error.log 2>/dev/null
echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
sccache --show-stats
sccache --stop-server || true
echo "::endgroup::"
}
trap_add sccache_epilogue EXIT
fi
if which ccache > /dev/null; then
# Report ccache stats for easier debugging
ccache --zero-stats
ccache --show-stats
function ccache_epilogue() {
ccache --show-stats
}
trap_add ccache_epilogue EXIT
fi
fi

View file

@ -5,9 +5,6 @@
source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
set -ex
# Save the SCRIPT_DIR absolute path in case later we chdir (as occurs in the gpu perf test)
SCRIPT_DIR="$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )"
# Required environment variables:
# $BUILD_ENVIRONMENT (should be set by your Docker image)
@ -22,56 +19,6 @@ if [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
export HSA_FORCE_FINE_GRAIN_PCIE=1
fi
if [[ "$BUILD_ENVIRONMENT" != *win-* ]]; then
if which sccache > /dev/null; then
# Save sccache logs to file
sccache --stop-server > /dev/null 2>&1 || true
rm -f ~/sccache_error.log || true
if [[ -n "${SKIP_SCCACHE_INITIALIZATION:-}" ]]; then
# sccache --start-server seems to hang forever on self hosted runners for GHA
# so let's just go ahead and skip the --start-server altogether since it seems
# as though sccache still gets used even when the sscache server isn't started
# explicitly
echo "Skipping sccache server initialization, setting environment variables"
export SCCACHE_IDLE_TIMEOUT=1200
export SCCACHE_ERROR_LOG=~/sccache_error.log
export RUST_LOG=sccache::server=error
elif [[ "${BUILD_ENVIRONMENT}" == *rocm* ]]; then
SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=0 sccache --start-server
else
# increasing SCCACHE_IDLE_TIMEOUT so that extension_backend_test.cpp can build after this PR:
# https://github.com/pytorch/pytorch/pull/16645
SCCACHE_ERROR_LOG=~/sccache_error.log SCCACHE_IDLE_TIMEOUT=1200 RUST_LOG=sccache::server=error sccache --start-server
fi
# Report sccache stats for easier debugging
sccache --zero-stats
function sccache_epilogue() {
echo "::group::Sccache Compilation Log"
echo '=================== sccache compilation log ==================='
python "$SCRIPT_DIR/print_sccache_log.py" ~/sccache_error.log 2>/dev/null
echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
sccache --show-stats
sccache --stop-server || true
echo "::endgroup::"
}
if [[ "${JOB_BASE_NAME}" == *-build ]]; then
trap_add sccache_epilogue EXIT
fi
fi
if which ccache > /dev/null; then
# Report ccache stats for easier debugging
ccache --zero-stats
ccache --show-stats
function ccache_epilogue() {
ccache --show-stats
}
trap_add ccache_epilogue EXIT
fi
fi
# TODO: Renable libtorch testing for MacOS, see https://github.com/pytorch/pytorch/issues/62598
# shellcheck disable=SC2034
BUILD_TEST_LIBTORCH=0

View file

@ -5,6 +5,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
echo "Testing pytorch docs"
cd "${SCRIPT_DIR}/../../docs"
cd docs
pip_install -r requirements.txt
make doctest

View file

@ -3,6 +3,8 @@
# shellcheck disable=SC2034
# shellcheck source=./macos-common.sh
source "$(dirname "${BASH_SOURCE[0]}")/macos-common.sh"
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
# Build PyTorch
if [ -z "${CI}" ]; then

View file

@ -424,6 +424,10 @@ test_torch_function_benchmark() {
}
build_xla() {
# xla test needs sccache setup.
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
XLA_DIR=xla
USE_CACHE=0
clone_pytorch_xla
@ -438,6 +442,10 @@ build_xla() {
}
test_xla() {
# xla test needs sccache setup.
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
clone_pytorch_xla
# shellcheck disable=SC1091
source "./xla/.circleci/common.sh"
@ -488,6 +496,10 @@ test_forward_backward_compatibility() {
test_bazel() {
set -e
# bazel test needs sccache setup.
# shellcheck source=./common-build.sh
source "$(dirname "${BASH_SOURCE[0]}")/common-build.sh"
get_bazel
# Test //c10/... without Google flags and logging libraries. The

View file

@ -12,6 +12,8 @@ fi
SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
# shellcheck source=./common.sh
source "$SCRIPT_PARENT_DIR/common.sh"
# shellcheck source=./common-build.sh
source "$SCRIPT_PARENT_DIR/common-build.sh"
IMAGE_COMMIT_ID=$(git rev-parse HEAD)
export IMAGE_COMMIT_ID

View file

@ -65,7 +65,7 @@ run_tests() {
done
"$SCRIPT_HELPERS_DIR"/test_python_shard.bat
if [[ ( -z "${JOB_BASE_NAME}" || "${JOB_BASE_NAME}" == *-test ) && $NUM_TEST_SHARDS -eq 1 ]]; then
if [[ $NUM_TEST_SHARDS -eq 1 ]]; then
"$SCRIPT_HELPERS_DIR"/test_custom_script_ops.bat
"$SCRIPT_HELPERS_DIR"/test_custom_backend.bat
"$SCRIPT_HELPERS_DIR"/test_libtorch.bat

View file

@ -785,7 +785,9 @@ def build_info() -> ReportMetaMeta:
os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", "HEAD"))
),
"build_branch": os.environ.get("BRANCH", os.environ.get("CIRCLE_BRANCH", "")),
"build_job": os.environ.get("JOB_BASE_NAME", os.environ.get("CIRCLE_JOB", "")),
"build_job": os.environ.get(
"BUILD_ENVIRONMENT", os.environ.get("CIRCLE_JOB", "")
),
"build_workflow_id": os.environ.get(
"WORKFLOW_ID", os.environ.get("CIRCLE_WORKFLOW_ID", "")
),
@ -877,7 +879,7 @@ def assemble_s3_object(
def send_report_to_s3(head_report: Version2Report) -> None:
job = os.getenv("JOB_BASE_NAME", os.environ.get("CIRCLE_JOB"))
job = os.getenv("BUILD_ENVIRONMENT", os.environ.get("CIRCLE_JOB"))
sha1 = os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", ""))
now = datetime.datetime.utcnow().isoformat()
@ -929,7 +931,7 @@ def print_regressions(head_report: Report, *, num_prev_commits: int) -> None:
else:
commits = commits[:-1]
job = os.environ.get("JOB_BASE_NAME", "")
job = os.environ.get("BUILD_ENVIRONMENT", "")
objects: Dict[Commit, List[Report]] = defaultdict(list)
for commit in commits:

View file

@ -21,15 +21,7 @@ class JobTimeJSON(TypedDict):
def _get_stripped_CI_job() -> str:
"""E.g. convert 'pytorch_windows_vs2019_py36_cuda10.1_build' to 'pytorch_windows_vs2019_py36_cuda10.1'."""
job = os.environ.get("JOB_BASE_NAME", "").rstrip("0123456789")
if job.endswith("_slow_test"):
job = job[: len(job) - len("_slow_test")]
elif job.endswith("_test") or job.endswith("-test"):
job = job[: len(job) - len("_test")]
elif job.endswith("_build") or job.endswith("-build"):
job = job[: len(job) - len("_build")]
return job
return os.environ.get("BUILD_ENVIRONMENT", "")
def _get_job_times_json(job_times: Dict[str, float]) -> JobTimeJSON: