From 06b52dd10346911850d4d2e404cefac787ec0bda Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 1 Mar 2024 23:08:10 +0000 Subject: [PATCH] TD outside of test job (#118250) Give TD it's own job so that each shard can get the results from this one job artifact and they will always be in sync with each other/no longer need to worry about consistently issues * Move test discovery to its own file that is not dependent on torch so it can be run without building torch * Cannot do cpp test discovery before building pytorch * Move TD calculation to own file that will create a json file with the final results * TD is now job/build env agnostic * TD will rank all tests, including those that test jobs may not want to run (ex it will rank distributed tests along with default tests, even though these tests are never run on the same machine together) Pull Request resolved: https://github.com/pytorch/pytorch/pull/118250 Approved by: https://github.com/huydhn --- .../actions/download-td-artifacts/action.yml | 29 +++++++ .github/workflows/_linux-test.yml | 4 + .github/workflows/_mac-test.yml | 6 ++ .github/workflows/_rocm-test.yml | 4 + .github/workflows/_win-test.yml | 4 + .github/workflows/periodic.yml | 23 +++++- .github/workflows/pull.yml | 39 ++++++++-- .github/workflows/rocm.yml | 11 ++- .github/workflows/slow.yml | 27 +++++-- .github/workflows/target_determination.yml | 70 +++++++++++++++++ .github/workflows/trunk.yml | 23 +++++- test/run_test.py | 41 +++------- tools/stats/export_test_times.py | 17 +---- tools/stats/upload_metrics.py | 4 +- tools/test/heuristics/test_interface.py | 26 +++++++ tools/test/test_upload_stats_lib.py | 7 +- .../testing/do_target_determination_for_s3.py | 76 +++++++++++++++++++ .../target_determination/determinator.py | 10 +-- .../heuristics/interface.py | 70 +++++++++++++++++ tools/testing/test_run.py | 20 ++++- 20 files changed, 433 insertions(+), 78 deletions(-) create mode 100644 .github/actions/download-td-artifacts/action.yml create mode 100644 .github/workflows/target_determination.yml create mode 100644 tools/testing/do_target_determination_for_s3.py diff --git a/.github/actions/download-td-artifacts/action.yml b/.github/actions/download-td-artifacts/action.yml new file mode 100644 index 00000000000..595093abaea --- /dev/null +++ b/.github/actions/download-td-artifacts/action.yml @@ -0,0 +1,29 @@ +name: Download TD Artifacts + +description: Download artifacts from target_determination.yml + +inputs: + use-gha: + description: If set to any value, use GHA to download the artifact. Otherwise use s3. + required: false + +runs: + using: composite + steps: + - name: Download TD Artifacts from S3 + if: ${{ !inputs.use-gha }} + uses: seemethere/download-artifact-s3@v4 + with: + name: td_results + + - name: Download TD Artifacts from GHA + if: inputs.use-gha + uses: actions/download-artifact@v3 + with: + name: td_results.json + + - name: Move artifacts to .additional_ci_files folder + shell: bash + run: | + mkdir -p .additional_ci_files + mv td_results.json .additional_ci_files/td_results.json diff --git a/.github/workflows/_linux-test.yml b/.github/workflows/_linux-test.yml index 1715d0cdcdb..9e41f8faa46 100644 --- a/.github/workflows/_linux-test.yml +++ b/.github/workflows/_linux-test.yml @@ -117,6 +117,10 @@ jobs: with: name: ${{ inputs.build-environment }} + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/_mac-test.yml b/.github/workflows/_mac-test.yml index 8d4091d14a0..b8e90771ec7 100644 --- a/.github/workflows/_mac-test.yml +++ b/.github/workflows/_mac-test.yml @@ -91,6 +91,12 @@ jobs: name: ${{ inputs.build-environment }} use-gha: true + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + with: + use-gha: true + - name: Setup miniconda uses: pytorch/test-infra/.github/actions/setup-miniconda@main with: diff --git a/.github/workflows/_rocm-test.yml b/.github/workflows/_rocm-test.yml index b807aca6f0f..1f2d86273ee 100644 --- a/.github/workflows/_rocm-test.yml +++ b/.github/workflows/_rocm-test.yml @@ -103,6 +103,10 @@ jobs: with: name: ${{ inputs.build-environment }} + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/_win-test.yml b/.github/workflows/_win-test.yml index a5a2548e080..ebc8434407a 100644 --- a/.github/workflows/_win-test.yml +++ b/.github/workflows/_win-test.yml @@ -114,6 +114,10 @@ jobs: run: | tree /F C:\$Env:GITHUB_RUN_ID\build-results + - name: Download TD artifacts + continue-on-error: true + uses: ./.github/actions/download-td-artifacts + - name: Get workflow job id id: get-job-id uses: ./.github/actions/get-workflow-job-id diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 18f7409fdf5..99f4dd99395 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -23,6 +23,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + parallelnative-linux-jammy-py3_8-gcc11-build: name: parallelnative-linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-build.yml @@ -39,7 +46,9 @@ jobs: parallelnative-linux-jammy-py3_8-gcc11-test: name: parallelnative-linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-test.yml - needs: parallelnative-linux-jammy-py3_8-gcc11-build + needs: + - parallelnative-linux-jammy-py3_8-gcc11-build + - target-determination with: build-environment: parallelnative-linux-jammy-py3.8-gcc11 docker-image: ${{ needs.parallelnative-linux-jammy-py3_8-gcc11-build.outputs.docker-image }} @@ -86,7 +95,9 @@ jobs: linux-focal-cuda11_8-py3_10-gcc9-debug-test: name: linux-focal-cuda11.8-py3.10-gcc9-debug uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda11_8-py3_10-gcc9-debug-build + needs: + - linux-focal-cuda11_8-py3_10-gcc9-debug-build + - target-determination with: build-environment: linux-focal-cuda11.8-py3.10-gcc9-debug docker-image: ${{ needs.linux-focal-cuda11_8-py3_10-gcc9-debug-build.outputs.docker-image }} @@ -110,7 +121,9 @@ jobs: win-vs2019-cuda11_8-py3-test: name: win-vs2019-cuda11.8-py3 uses: ./.github/workflows/_win-test.yml - needs: win-vs2019-cuda11_8-py3-build + needs: + - win-vs2019-cuda11_8-py3-build + - target-determination with: build-environment: win-vs2019-cuda11.8-py3 cuda-version: "11.8" @@ -214,7 +227,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 214463ec823..1e07c638975 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -20,6 +20,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-jammy-py3_8-gcc11-build: name: linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-build.yml @@ -41,7 +48,9 @@ jobs: linux-jammy-py3_8-gcc11-test: name: linux-jammy-py3.8-gcc11 uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_8-gcc11-build + needs: + - linux-jammy-py3_8-gcc11-build + - target-determination with: build-environment: linux-jammy-py3.8-gcc11 docker-image: ${{ needs.linux-jammy-py3_8-gcc11-build.outputs.docker-image }} @@ -97,7 +106,9 @@ jobs: linux-jammy-py3_10-clang15-asan-test: name: linux-jammy-py3.10-clang15-asan uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_10-clang15-asan-build + needs: + - linux-jammy-py3_10-clang15-asan-build + - target-determination with: build-environment: linux-jammy-py3.10-clang15-asan docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }} @@ -119,7 +130,9 @@ jobs: linux-focal-py3_8-clang10-onnx-test: name: linux-focal-py3.8-clang10-onnx uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-onnx-build + needs: + - linux-focal-py3_8-clang10-onnx-build + - target-determination with: build-environment: linux-focal-py3.8-clang10-onnx docker-image: ${{ needs.linux-focal-py3_8-clang10-onnx-build.outputs.docker-image }} @@ -146,7 +159,9 @@ jobs: linux-focal-py3_8-clang10-test: name: linux-focal-py3.8-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-build + needs: + - linux-focal-py3_8-clang10-build + - target-determination with: build-environment: linux-focal-py3.8-clang10 docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }} @@ -173,7 +188,9 @@ jobs: linux-focal-py3_11-clang10-test: name: linux-focal-py3.11-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_11-clang10-build + needs: + - linux-focal-py3_11-clang10-build + - target-determination with: build-environment: linux-focal-py3.11-clang10 docker-image: ${{ needs.linux-focal-py3_11-clang10-build.outputs.docker-image }} @@ -218,7 +235,9 @@ jobs: linux-focal-cuda11_8-py3_10-gcc9-test: name: linux-focal-cuda11.8-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda11_8-py3_10-gcc9-build + needs: + - linux-focal-cuda11_8-py3_10-gcc9-build + - target-determination with: timeout-minutes: 360 build-environment: linux-focal-cuda11.8-py3.10-gcc9 @@ -244,7 +263,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-test: name: linux-focal-cuda12.1-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-build + - target-determination with: timeout-minutes: 360 build-environment: linux-focal-cuda12.1-py3.10-gcc9 @@ -415,7 +436,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-sm86-test: name: linux-focal-cuda12.1-py3.10-gcc9-sm86 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-sm86-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }} diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index d724fdb37d1..24542c3ddc4 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -18,6 +18,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-focal-rocm6_0-py3_8-build: name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_linux-build.yml @@ -41,7 +48,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/.github/workflows/slow.yml b/.github/workflows/slow.yml index 9c3a52f7537..33577986f64 100644 --- a/.github/workflows/slow.yml +++ b/.github/workflows/slow.yml @@ -21,6 +21,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build: name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck uses: ./.github/workflows/_linux-build.yml @@ -39,7 +46,9 @@ jobs: linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-test: name: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build + needs: + - linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3-gcc9-slow-gradcheck docker-image: ${{ needs.linux-focal-cuda12_1-py3-gcc9-slow-gradcheck-build.outputs.docker-image }} @@ -62,7 +71,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-sm86-test: name: linux-focal-cuda12.1-py3.10-gcc9-sm86 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-sm86-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-sm86-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9-sm86 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-sm86-build.outputs.docker-image }} @@ -82,7 +93,9 @@ jobs: linux-focal-py3_8-clang10-test: name: linux-focal-py3.8-clang10 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-py3_8-clang10-build + needs: + - linux-focal-py3_8-clang10-build + - target-determination with: build-environment: linux-focal-py3.8-clang10 docker-image: ${{ needs.linux-focal-py3_8-clang10-build.outputs.docker-image }} @@ -105,7 +118,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} @@ -127,7 +142,9 @@ jobs: linux-jammy-py3_10-clang15-asan-test: name: linux-jammy-py3.10-clang15-asan uses: ./.github/workflows/_linux-test.yml - needs: linux-jammy-py3_10-clang15-asan-build + needs: + - linux-jammy-py3_10-clang15-asan-build + - target-determination with: build-environment: linux-jammy-py3.10-clang15-asan docker-image: ${{ needs.linux-jammy-py3_10-clang15-asan-build.outputs.docker-image }} diff --git a/.github/workflows/target_determination.yml b/.github/workflows/target_determination.yml new file mode 100644 index 00000000000..53027259832 --- /dev/null +++ b/.github/workflows/target_determination.yml @@ -0,0 +1,70 @@ +name: target-determination + +on: + workflow_call: + +jobs: + target-determination: + # Don't run on forked repos + if: github.repository_owner == 'pytorch' + runs-on: linux.2xlarge + steps: + # [pytorch repo ref] + # Use a pytorch/pytorch reference instead of a reference to the local + # checkout because when we run this action we don't *have* a local + # checkout. In other cases you should prefer a local checkout. + - name: Checkout PyTorch + uses: pytorch/pytorch/.github/actions/checkout-pytorch@main + with: + submodules: false + + - name: Setup Linux + uses: ./.github/actions/setup-linux + + - name: Get workflow job id + id: get-job-id + uses: ./.github/actions/get-workflow-job-id + if: always() + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download pytest cache + uses: ./.github/actions/pytest-cache-download + continue-on-error: true + with: + cache_dir: .pytest_cache + job_identifier: ${{ github.workflow }} + + - name: Do TD + id: td + continue-on-error: true + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_WORKFLOW: ${{ github.workflow }} + GITHUB_JOB: ${{ github.job }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id }} + JOB_NAME: ${{ steps.get-job-id.outputs.job-name }} + run: | + python3 -m pip install boto3==1.19.12 + python3 tools/testing/do_target_determination_for_s3.py + + - name: Upload TD results to s3 + uses: seemethere/upload-artifact-s3@v5 + if: steps.td.outcome == 'success' + with: + name: td_results + retention-days: 14 + if-no-files-found: error + path: td_results.json + + - name: Store TD results on GHA + uses: actions/upload-artifact@v3 + if: steps.td.outcome == 'success' + with: + name: td_results.json + retention-days: 14 + if-no-files-found: error + path: td_results.json diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index b6a3f09c86e..c0538a8600d 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -19,6 +19,13 @@ concurrency: permissions: read-all jobs: + target-determination: + name: before-test + uses: ./.github/workflows/target_determination.yml + permissions: + id-token: write + contents: read + # Build PyTorch with BUILD_CAFFE2=ON caffe2-linux-jammy-py3_8-gcc11-build: name: caffe2-linux-jammy-py3.8-gcc11 @@ -47,7 +54,9 @@ jobs: linux-focal-cuda12_1-py3_10-gcc9-test: name: linux-focal-cuda12.1-py3.10-gcc9 uses: ./.github/workflows/_linux-test.yml - needs: linux-focal-cuda12_1-py3_10-gcc9-build + needs: + - linux-focal-cuda12_1-py3_10-gcc9-build + - target-determination with: build-environment: linux-focal-cuda12.1-py3.10-gcc9 docker-image: ${{ needs.linux-focal-cuda12_1-py3_10-gcc9-build.outputs.docker-image }} @@ -128,7 +137,9 @@ jobs: macos-12-py3-arm64-test: name: macos-12-py3-arm64 uses: ./.github/workflows/_mac-test.yml - needs: macos-12-py3-arm64-build + needs: + - macos-12-py3-arm64-build + - target-determination with: build-environment: macos-12-py3-arm64 # Same as the build job @@ -153,7 +164,9 @@ jobs: win-vs2019-cpu-py3-test: name: win-vs2019-cpu-py3 uses: ./.github/workflows/_win-test.yml - needs: win-vs2019-cpu-py3-build + needs: + - win-vs2019-cpu-py3-build + - target-determination with: build-environment: win-vs2019-cpu-py3 cuda-version: cpu @@ -195,7 +208,9 @@ jobs: contents: read name: linux-focal-rocm6.0-py3.8 uses: ./.github/workflows/_rocm-test.yml - needs: linux-focal-rocm6_0-py3_8-build + needs: + - linux-focal-rocm6_0-py3_8-build + - target-determination with: build-environment: linux-focal-rocm6.0-py3.8 docker-image: ${{ needs.linux-focal-rocm6_0-py3_8-build.outputs.docker-image }} diff --git a/test/run_test.py b/test/run_test.py index 33b7d5805fe..4b0fbc872e6 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -54,11 +54,7 @@ from tools.testing.discover_tests import ( parse_test_module, TESTS, ) -from tools.testing.target_determination.determinator import ( - AggregatedHeuristics, - get_prediction_confidences, - get_test_prioritizations, -) +from tools.testing.do_target_determination_for_s3 import import_results from tools.testing.test_run import TestRun from tools.testing.test_selections import ( @@ -1631,24 +1627,17 @@ def main(): test_directory = str(REPO_ROOT / "test") selected_tests = get_selected_tests(options) + test_prioritizations = import_results() + test_prioritizations.amend_tests(selected_tests) + os.makedirs(REPO_ROOT / "test" / "test-reports", exist_ok=True) if options.coverage and not PYTORCH_COLLECT_COVERAGE: shell(["coverage", "erase"]) - aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests) - - with open( - REPO_ROOT / "test" / "test-reports" / "td_heuristic_rankings.log", "a" - ) as f: - if IS_CI: - # downloading test cases configuration to local environment - get_test_case_configs(dirpath=test_directory) - aggregated_heuristics = get_test_prioritizations(selected_tests, file=f) - - test_prioritizations = aggregated_heuristics.get_aggregated_priorities() - - f.write(test_prioritizations.get_info_str()) + if IS_CI: + # downloading test cases configuration to local environment + get_test_case_configs(dirpath=test_directory) test_file_times_dict = load_test_file_times() test_class_times_dict = load_test_class_times() @@ -1736,21 +1725,15 @@ def main(): all_failures = test_batch.failures if IS_CI: - num_tests = len(selected_tests) for test, _ in all_failures: - test_stats = aggregated_heuristics.get_test_stats(test) - test_stats["num_total_tests"] = num_tests - - print_to_stderr("Emiting td_test_failure_stats") + test_stats = test_prioritizations.get_test_stats(test) + print_to_stderr("Emiting td_test_failure_stats_v2") emit_metric( - "td_test_failure_stats", + "td_test_failure_stats_v2", { - **test_stats, - "confidence_ratings": get_prediction_confidences( - selected_tests - ), + "selected_tests": selected_tests, "failure": str(test), - "tests": selected_tests, + **test_stats, }, ) diff --git a/tools/stats/export_test_times.py b/tools/stats/export_test_times.py index 8254157f0d7..2b9c0c45068 100644 --- a/tools/stats/export_test_times.py +++ b/tools/stats/export_test_times.py @@ -3,26 +3,13 @@ import sys REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent sys.path.append(str(REPO_ROOT)) -from tools.stats.import_test_stats import ( - copy_pytest_cache, - get_td_heuristic_historial_edited_files_json, - get_td_heuristic_profiling_json, - get_test_class_ratings, - get_test_class_times, - get_test_file_ratings, - get_test_times, -) +from tools.stats.import_test_stats import get_test_class_times, get_test_times def main() -> None: - print("Exporting files from test-infra") + print("Exporting test times from test-infra") get_test_times() get_test_class_times() - get_test_file_ratings() - get_test_class_ratings() - get_td_heuristic_historial_edited_files_json() - get_td_heuristic_profiling_json() - copy_pytest_cache() if __name__ == "__main__": diff --git a/tools/stats/upload_metrics.py b/tools/stats/upload_metrics.py index b4ff5256fb9..8a0e93858b6 100644 --- a/tools/stats/upload_metrics.py +++ b/tools/stats/upload_metrics.py @@ -105,7 +105,7 @@ def emit_metric( env_var_metrics = [ EnvVarMetric("repo", "GITHUB_REPOSITORY"), EnvVarMetric("workflow", "GITHUB_WORKFLOW"), - EnvVarMetric("build_environment", "BUILD_ENVIRONMENT"), + EnvVarMetric("build_environment", "BUILD_ENVIRONMENT", required=False), EnvVarMetric("job", "GITHUB_JOB"), EnvVarMetric("test_config", "TEST_CONFIG", required=False), EnvVarMetric("pr_number", "PR_NUMBER", required=False, type_conversion_fn=int), @@ -177,6 +177,8 @@ def _convert_float_values_to_decimals(data: Dict[str, Any]) -> Dict[str, Any]: return [_helper(v) for v in o] if isinstance(o, dict): return {_helper(k): _helper(v) for k, v in o.items()} + if isinstance(o, tuple): + return tuple(_helper(v) for v in o) return o return {k: _helper(v) for k, v in data.items()} diff --git a/tools/test/heuristics/test_interface.py b/tools/test/heuristics/test_interface.py index f507640af65..df122ab7d56 100644 --- a/tools/test/heuristics/test_interface.py +++ b/tools/test/heuristics/test_interface.py @@ -540,5 +540,31 @@ class TestAggregatedHeuristicsTestStats(TestTD): aggregator.get_test_stats(TestRun("test2")) +class TestJsonParsing(TestTD): + def test_json_parsing_matches_TestPrioritizations(self) -> None: + tests = ["test1", "test2", "test3", "test4", "test5"] + tp = interface.TestPrioritizations( + tests, + { + TestRun("test3", included=["ClassA"]): 0.8, + TestRun("test3", excluded=["ClassA"]): 0.2, + TestRun("test4"): 0.7, + TestRun("test5"): 0.6, + }, + ) + tp_json = tp.to_json() + tp_json_to_tp = interface.TestPrioritizations.from_json(tp_json) + + self.assertSetEqual(tp._original_tests, tp_json_to_tp._original_tests) + self.assertDictEqual(tp._test_scores, tp_json_to_tp._test_scores) + + def test_json_parsing_matches_TestRun(self) -> None: + testrun = TestRun("test1", included=["classA", "classB"]) + testrun_json = testrun.to_json() + testrun_json_to_test = TestRun.from_json(testrun_json) + + self.assertTrue(testrun == testrun_json_to_test) + + if __name__ == "__main__": unittest.main() diff --git a/tools/test/test_upload_stats_lib.py b/tools/test/test_upload_stats_lib.py index 8199994d614..0baf323966e 100644 --- a/tools/test/test_upload_stats_lib.py +++ b/tools/test/test_upload_stats_lib.py @@ -1,13 +1,18 @@ import decimal import inspect +import pathlib +import sys import unittest from typing import Any, Dict from unittest import mock +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT)) from tools.stats.upload_metrics import add_global_metric, emit_metric from tools.stats.upload_stats_lib import BATCH_SIZE, upload_to_rockset +sys.path.remove(str(REPO_ROOT)) # default values REPO = "some/repo" @@ -278,7 +283,7 @@ class TestUploadStats(unittest.TestCase): mock.patch.dict( "os.environ", { - "BUILD_ENVIRONMENT": "", + "GITHUB_JOB": "", }, ).start() diff --git a/tools/testing/do_target_determination_for_s3.py b/tools/testing/do_target_determination_for_s3.py new file mode 100644 index 00000000000..c7691b56792 --- /dev/null +++ b/tools/testing/do_target_determination_for_s3.py @@ -0,0 +1,76 @@ +import json +import os +import pathlib +import sys + + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from tools.stats.import_test_stats import ( + copy_pytest_cache, + get_td_heuristic_historial_edited_files_json, + get_td_heuristic_profiling_json, + get_test_class_ratings, + get_test_class_times, + get_test_file_ratings, + get_test_times, +) +from tools.stats.upload_metrics import emit_metric + +from tools.testing.discover_tests import TESTS +from tools.testing.target_determination.determinator import ( + AggregatedHeuristics, + get_test_prioritizations, + TestPrioritizations, +) + +sys.path.remove(str(REPO_ROOT)) + + +def import_results() -> TestPrioritizations: + if not (REPO_ROOT / ".additional_ci_files/td_results.json").exists(): + print("No TD results found") + return TestPrioritizations([], {}) + with open(REPO_ROOT / ".additional_ci_files/td_results.json") as f: + td_results = json.load(f) + tp = TestPrioritizations.from_json(td_results) + + return tp + + +def main() -> None: + selected_tests = TESTS + + aggregated_heuristics: AggregatedHeuristics = AggregatedHeuristics(selected_tests) + + get_test_times() + get_test_class_times() + get_test_file_ratings() + get_test_class_ratings() + get_td_heuristic_historial_edited_files_json() + get_td_heuristic_profiling_json() + copy_pytest_cache() + + aggregated_heuristics = get_test_prioritizations(selected_tests) + + test_prioritizations = aggregated_heuristics.get_aggregated_priorities() + + if os.getenv("CI") == "true": + print("Emitting metrics") + # Split into 3 due to size constraints + emit_metric( + "td_results_final_test_prioritizations", + {"test_prioritizations": test_prioritizations.to_json()}, + ) + emit_metric( + "td_results_aggregated_heuristics", + {"aggregated_heuristics": aggregated_heuristics.to_json()}, + ) + + with open(REPO_ROOT / "td_results.json", "w") as f: + f.write(json.dumps(test_prioritizations.to_json())) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/target_determination/determinator.py b/tools/testing/target_determination/determinator.py index ea99942c27d..35994758f38 100644 --- a/tools/testing/target_determination/determinator.py +++ b/tools/testing/target_determination/determinator.py @@ -1,5 +1,5 @@ import sys -from typing import Any, Dict, List +from typing import Any, List from tools.testing.target_determination.heuristics import ( AggregatedHeuristics as AggregatedHeuristics, @@ -23,11 +23,3 @@ def get_test_prioritizations( print(new_rankings.get_info_str(), file=file) return aggregated_results - - -def get_prediction_confidences(tests: List[str]) -> Dict[str, TestPrioritizations]: - # heuristic name -> test -> rating/confidence - rankings: Dict[str, TestPrioritizations] = {} - for heuristic in HEURISTICS: - rankings[heuristic.name] = heuristic.get_prediction_confidence(tests) - return rankings diff --git a/tools/testing/target_determination/heuristics/interface.py b/tools/testing/target_determination/heuristics/interface.py index c6935643ce5..a5dae08f74c 100644 --- a/tools/testing/target_determination/heuristics/interface.py +++ b/tools/testing/target_determination/heuristics/interface.py @@ -139,6 +139,66 @@ class TestPrioritizations: return {"position": idx, "score": score} raise AssertionError(f"Test run {test_run} not found") + def get_test_stats(self, test: TestRun) -> Dict[str, Any]: + return { + "test_name": test.test_file, + "test_filters": test.get_pytest_filter(), + **self.get_priority_info_for_test(test), + "max_score": max(score for score, _ in self._traverse_scores()), + "min_score": min(score for score, _ in self._traverse_scores()), + "all_scores": { + str(test): score for test, score in self._test_scores.items() + }, + } + + def to_json(self) -> Dict[str, Any]: + """ + Returns a JSON dict that describes this TestPrioritizations object. + """ + json_dict = { + "_test_scores": [ + (test.to_json(), score) + for test, score in self._test_scores.items() + if score != 0 + ], + "_original_tests": list(self._original_tests), + } + return json_dict + + @staticmethod + def from_json(json_dict: Dict[str, Any]) -> "TestPrioritizations": + """ + Returns a TestPrioritizations object from a JSON dict. + """ + test_prioritizations = TestPrioritizations( + tests_being_ranked=json_dict["_original_tests"], + scores={ + TestRun.from_json(testrun_json): score + for testrun_json, score in json_dict["_test_scores"] + }, + ) + return test_prioritizations + + def amend_tests(self, tests: List[str]) -> None: + """ + Removes tests that are not in the given list from the + TestPrioritizations. Adds tests that are in the list but not in the + TestPrioritizations. + """ + valid_scores = { + test: score + for test, score in self._test_scores.items() + if test.test_file in tests + } + self._test_scores = valid_scores + + for test in tests: + if test not in self._original_tests: + self._test_scores[TestRun(test)] = 0 + self._original_tests = frozenset(tests) + + self.validate() + class AggregatedHeuristics: """ @@ -224,6 +284,16 @@ class AggregatedHeuristics: return stats + def to_json(self) -> Dict[str, Any]: + """ + Returns a JSON dict that describes this AggregatedHeuristics object. + """ + json_dict: Dict[str, Any] = {} + for heuristic, heuristic_results in self._heuristic_results.items(): + json_dict[heuristic.name] = heuristic_results.to_json() + + return json_dict + class HeuristicInterface: """ diff --git a/tools/testing/test_run.py b/tools/testing/test_run.py index c5b04e57d34..25429b3c2c3 100644 --- a/tools/testing/test_run.py +++ b/tools/testing/test_run.py @@ -1,6 +1,6 @@ from copy import copy from functools import total_ordering -from typing import FrozenSet, Iterable, List, Optional, Union +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Union class TestRun: @@ -210,6 +210,24 @@ class TestRun: return (self | other) - (self - other) - (other - self) + def to_json(self) -> Dict[str, Any]: + r: Dict[str, Any] = { + "test_file": self.test_file, + } + if self._included: + r["included"] = list(self._included) + if self._excluded: + r["excluded"] = list(self._excluded) + return r + + @staticmethod + def from_json(json: Dict[str, Any]) -> "TestRun": + return TestRun( + json["test_file"], + included=json.get("included", []), + excluded=json.get("excluded", []), + ) + @total_ordering class ShardedTest: