saymrwulf/pytorch
1
0
Fork 0
mirror of https://github.com/saymrwulf/pytorch.git synced 2026-05-14 20:57:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cleanup old TD logic (#68842)

Browse source 
Summary:
Remove `--determine-from` option from run_test.py and remove all
references from corresponding test scripts

Followup after https://github.com/pytorch/pytorch/pull/64921

Pull Request resolved: https://github.com/pytorch/pytorch/pull/68842

Reviewed By: seemethere, janeyx99

Differential Revision: D32631418

Pulled By: malfet

fbshipit-source-id: bdb5dd888c1d97dfaf95c1f299bf8073f3de9588
This commit is contained in:
Nikita Shulga 2021-11-23 18:44:14 -08:00 committed by Facebook GitHub Bot
parent d9f3feb5a2
commit b5b62b3408
9 changed files with 23 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.jenkins/pytorch/macos-test.sh
View file

@ -45,20 +45,10 @@ test_python_all() {
export GLOO_SOCKET_IFNAME=lo0
echo "Ninja version: $(ninja --version)"
# Try to pull value from CIRCLE_PULL_REQUEST first then GITHUB_HEAD_REF second
# CIRCLE_PULL_REQUEST comes from CircleCI
# NOTE: file_diff_from_base is currently bugged for GHA due to an issue finding a merge base for ghstack PRs
# see https://github.com/pytorch/pytorch/issues/60111
IN_PULL_REQUEST=${CIRCLE_PULL_REQUEST:-${GITHUB_HEAD_REF:-}}
if [ -n "$IN_PULL_REQUEST" ]; then
DETERMINE_FROM=$(mktemp)
file_diff_from_base "$DETERMINE_FROM"
fi
# Increase default limit on open file handles from 256 to 1024
ulimit -n 1024
python test/run_test.py --verbose --exclude-jit-executor --determine-from="$DETERMINE_FROM"
python test/run_test.py --verbose --exclude-jit-executor
assert_git_not_dirty
}

2
.jenkins/pytorch/run_glootls_test.sh
View file

@ -10,7 +10,7 @@ export GLOO_DEVICE_TRANSPORT_TCP_TLS_PKEY=${TMP_CERT_DIR}/pkey.key
export GLOO_DEVICE_TRANSPORT_TCP_TLS_CERT=${TMP_CERT_DIR}/cert.pem
export GLOO_DEVICE_TRANSPORT_TCP_TLS_CA_FILE=${TMP_CERT_DIR}/ca.pem
time python test/run_test.py --include distributed/test_c10d_gloo --verbose --determine-from="$DETERMINE_FROM" -- ProcessGroupGlooTest
time python test/run_test.py --include distributed/test_c10d_gloo --verbose -- ProcessGroupGlooTest
unset GLOO_DEVICE_TRANSPORT
unset GLOO_DEVICE_TRANSPORT_TCP_TLS_PKEY

18
.jenkins/pytorch/test.sh
View file

@ -153,16 +153,8 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX512-* || $TEST_CONFIG == 'nogpu_NO_AVX
export ATEN_CPU_CAPABILITY=avx2
fi
# if PR_NUMBER exist, use it to grab PR contents.
DETERMINE_FROM=$(mktemp)
if [ -n "$PR_NUMBER" ]; then
get_pr_change_files "$PR_NUMBER" "$DETERMINE_FROM"
else
file_diff_from_base "$DETERMINE_FROM"
fi
test_python_legacy_jit() {
time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="$DETERMINE_FROM"
time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose
assert_git_not_dirty
}
@ -171,12 +163,12 @@ test_python_shard() {
echo "NUM_TEST_SHARDS must be defined to run a Python test shard"
exit 1
fi
time python test/run_test.py --exclude-jit-executor --exclude-distributed-tests --shard "$1" "$NUM_TEST_SHARDS" --verbose --determine-from="$DETERMINE_FROM"
time python test/run_test.py --exclude-jit-executor --exclude-distributed-tests --shard "$1" "$NUM_TEST_SHARDS" --verbose
assert_git_not_dirty
}
test_python() {
time python test/run_test.py --exclude-jit-executor --exclude-distributed-tests --verbose --determine-from="$DETERMINE_FROM"
time python test/run_test.py --exclude-jit-executor --exclude-distributed-tests --verbose
assert_git_not_dirty
}
@ -317,7 +309,7 @@ test_vulkan() {
test_distributed() {
echo "Testing distributed python tests"
time python test/run_test.py --distributed-tests --verbose --determine-from="$DETERMINE_FROM"
time python test/run_test.py --distributed-tests --verbose
assert_git_not_dirty
if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
@ -476,7 +468,7 @@ test_benchmarks() {
test_cpp_extensions() {
# This is to test whether cpp extension build is compatible with current env. No need to test both ninja and no-ninja build
time python test/run_test.py --include test_cpp_extensions_aot_ninja --verbose --determine-from="$DETERMINE_FROM"
time python test/run_test.py --include test_cpp_extensions_aot_ninja --verbose
assert_git_not_dirty
}

4
.jenkins/pytorch/win-test-helpers/test_python.bat
View file

@ -5,9 +5,9 @@ if "%RUN_SMOKE_TESTS_ONLY%"=="1" (
curl --retry 3 -k https://raw.githubusercontent.com/pytorch/test-infra/main/stats/windows_smoke_tests.csv --output .pytorch_specified_test_cases.csv
if ERRORLEVEL 1 exit /b 1
python run_test.py --exclude-jit-executor --verbose --determine-from="%1" --run-specified-test-cases
python run_test.py --exclude-jit-executor --verbose --run-specified-test-cases
) else (
python run_test.py --exclude-jit-executor --verbose --determine-from="%1"
python run_test.py --exclude-jit-executor --verbose
)
popd
if ERRORLEVEL 1 exit /b 1

4
.jenkins/pytorch/win-test-helpers/test_python_first_shard.bat
View file

@ -23,9 +23,9 @@ if "%RUN_SMOKE_TESTS_ONLY%"=="1" (
curl --retry 3 -k https://raw.githubusercontent.com/pytorch/test-infra/main/stats/windows_smoke_tests.csv --output .pytorch_specified_test_cases.csv
if ERRORLEVEL 1 exit /b 1
python run_test.py --exclude-jit-executor --shard 1 2 --verbose --determine-from="%1" --run-specified-test-cases
python run_test.py --exclude-jit-executor --shard 1 2 --verbose --run-specified-test-cases
) else (
python run_test.py --exclude-jit-executor --shard 1 2 --verbose --determine-from="%1"
python run_test.py --exclude-jit-executor --shard 1 2 --verbose
)
if ERRORLEVEL 1 exit /b 1

2
.jenkins/pytorch/win-test-helpers/test_python_jit_legacy.bat
View file

@ -6,7 +6,7 @@ copy /Y "%PYTORCH_FINAL_PACKAGE_DIR_WIN%\.pytorch-test-times.json" "%TEST_DIR_WI
pushd test
echo Run jit_profiling tests
python run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose --determine-from="%1"
python run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose
if ERRORLEVEL 1 exit /b 1
popd

4
.jenkins/pytorch/win-test-helpers/test_python_second_shard.bat
View file

@ -10,9 +10,9 @@ if "%RUN_SMOKE_TESTS_ONLY%"=="1" (
curl --retry 3 -k https://raw.githubusercontent.com/pytorch/test-infra/main/stats/windows_smoke_tests.csv --output .pytorch_specified_test_cases.csv
if ERRORLEVEL 1 exit /b 1
python run_test.py --exclude-jit-executor --shard 2 2 --verbose --determine-from="%1" --run-specified-test-cases
python run_test.py --exclude-jit-executor --shard 2 2 --verbose --run-specified-test-cases
) else (
python run_test.py --exclude-jit-executor --shard 2 2 --verbose --determine-from="%1"
python run_test.py --exclude-jit-executor --shard 2 2 --verbose
)
popd

17
.jenkins/pytorch/win-test.sh
View file

@ -42,15 +42,6 @@ fi
export SCRIPT_HELPERS_DIR=$SCRIPT_PARENT_DIR/win-test-helpers
# Try to pull value from CIRCLE_PULL_REQUEST
# NOTE: file_diff_from_base is currently bugged for GHA due to an issue finding a merge base for ghstack PRs
# see https://github.com/pytorch/pytorch/issues/60111
IN_PULL_REQUEST=${CIRCLE_PULL_REQUEST:-}
if [ -n "$IN_PULL_REQUEST" ]; then
DETERMINE_FROM="${TMP_DIR}/determine_from"
file_diff_from_base "$DETERMINE_FROM"
fi
if [[ "${BUILD_ENVIRONMENT}" == *cuda11* ]]; then
export BUILD_SPLIT_CUDA=ON
fi
@ -72,7 +63,7 @@ run_tests() {
done
if [[ ( -z "${JOB_BASE_NAME}" || "${JOB_BASE_NAME}" == *-test ) && $NUM_TEST_SHARDS -eq 1 ]]; then
"$SCRIPT_HELPERS_DIR"/test_python.bat "$DETERMINE_FROM"
"$SCRIPT_HELPERS_DIR"/test_python.bat
if [[ -z ${RUN_SMOKE_TESTS_ONLY} ]]; then
"$SCRIPT_HELPERS_DIR"/test_custom_script_ops.bat
@ -81,17 +72,17 @@ run_tests() {
fi
else
if [[ "${JOB_BASE_NAME}" == *-test1 || ("${SHARD_NUMBER}" == 1 && $NUM_TEST_SHARDS -gt 1) ]]; then
"$SCRIPT_HELPERS_DIR"/test_python_first_shard.bat "$DETERMINE_FROM"
"$SCRIPT_HELPERS_DIR"/test_python_first_shard.bat
if [[ -z ${RUN_SMOKE_TESTS_ONLY} ]]; then
"$SCRIPT_HELPERS_DIR"/test_libtorch.bat
if [[ "${USE_CUDA}" == "1" ]]; then
"$SCRIPT_HELPERS_DIR"/test_python_jit_legacy.bat "$DETERMINE_FROM"
"$SCRIPT_HELPERS_DIR"/test_python_jit_legacy.bat
fi
fi
elif [[ "${JOB_BASE_NAME}" == *-test2 || ("${SHARD_NUMBER}" == 2 && $NUM_TEST_SHARDS -gt 1) ]]; then
"$SCRIPT_HELPERS_DIR"/test_python_second_shard.bat "$DETERMINE_FROM"
"$SCRIPT_HELPERS_DIR"/test_python_second_shard.bat
if [[ -z ${RUN_SMOKE_TESTS_ONLY} ]]; then
"$SCRIPT_HELPERS_DIR"/test_custom_backend.bat

9
test/run_test.py
View file

@ -724,10 +724,11 @@ def parse_args():
action="store_true",
help="always run blocklisted windows tests",
)
parser.add_argument(
"--determine-from",
help="File of affected source filenames to determine which tests to run.",
)
# NS: Disable target determination until it can be made more reliable
# parser.add_argument(
# "--determine-from",
# help="File of affected source filenames to determine which tests to run.",
# )
parser.add_argument(
"--continue-through-error",
action="store_true",