mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-14 20:57:59 +00:00
Fix scribe logs again (#61768)
Summary:
revert the revert of 3624d75 with additional fix in https://github.com/pytorch/pytorch/pull/61764
Got the corrent logs sent to lambda
```
...
,"21721":"OK","21722":"OK","21723":"OK","21724":"OK","21725":"OK","21726":"OK","21727":"OK","21728":"OK","21729":"OK","21730":"OK","21731":"OK","21732":"OK","21733":"OK","21734":"OK","21735":"OK","21736":"OK","21737":"OK","21738":"OK","21739":"OK","21740":"OK","21741":"OK","21742":"OK","21743":"OK","21744":"OK","21745":"OK","21746":"OK","21747":"OK","21748":"OK","21749":"OK","21750":"OK","21751":"OK","21752":"OK","21753":"OK","21754":"OK","21755":"OK","21756":"OK","21757":"OK","21758":"OK","21759":"OK","21760":"OK","21761":"OK","21762":"OK","21763":"OK","21764":"OK","21765":"OK","21766":"OK","21767":"OK","21768":"OK","21769":"OK","21770":"OK","21771":"OK","21772":"OK","21773":"OK","21774":"OK","21775":"OK","21776":"OK","21777":"OK","21778":"OK","21779":"OK","21780":"OK","21781":"OK","21782":"OK","21783":"OK","21784":"OK","21785":"OK","21786":"OK","21787":"OK","21788":"OK","21789":"OK","21790":"OK","21791":"OK","21792":"OK","21793":"OK","21794":"OK","21795":"OK","21796":"OK","21797":"OK","21798":"OK","21799":"OK","21800":"OK","21801":"OK","21802":"OK","21803":"OK","21804":"OK","21805":"OK","21806":"OK","21807":"OK","21808":"OK","21809":"OK","21810":"OK","21811":"OK","21812":"OK","21813":"OK","21814":"OK","21815":"OK","21816":"OK","21817":"OK","21818":"OK","21819":"OK","21820":"OK","21821":"OK","21822":"OK","21823":"OK","21824":"OK","21825":"OK","21826":"OK"}}
class StartProcessesTest:
tests: 14 failed: 0 skipped: 0 errored: 0
run_time: 4.86 seconds
avg_time: 0.35 seconds
median_time: 0.01 seconds
3 longest tests:
test_function_large_ret_val time: 1.55 seconds
test_pcontext_wait time: 1.11 seconds
test_void_function time: 1.03 seconds
...
```
Pull Request resolved: https://github.com/pytorch/pytorch/pull/61768
Reviewed By: janeyx99
Differential Revision: D29735781
Pulled By: zhouzhuojie
fbshipit-source-id: 6882e334f5108d20773ad66d5300cd37eb509ded
This commit is contained in:
parent
311f1f275a
commit
59ca89dca8
25 changed files with 191 additions and 241 deletions
|
|
@ -212,7 +212,7 @@ commands:
|
|||
cd ~/project
|
||||
export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
|
||||
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 tools/stats/upload_binary_size_to_scuba.py android
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba android
|
||||
|
||||
##############################################################################
|
||||
# Binary build (nightlies nightly build) defaults
|
||||
|
|
@ -551,7 +551,7 @@ jobs:
|
|||
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 -mpip install requests && \
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
|
||||
python3 tools/stats/upload_binary_size_to_scuba.py || exit 0
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
- store_artifacts:
|
||||
path: /home/circleci/project/dist
|
||||
|
||||
|
|
@ -889,7 +889,7 @@ jobs:
|
|||
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 -mpip install requests && \
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
|
||||
python3 /pytorch/tools/stats/upload_binary_size_to_scuba.py || exit 0
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
- persist_to_workspace:
|
||||
root: /
|
||||
paths: final_pkgs
|
||||
|
|
|
|||
|
|
@ -171,4 +171,4 @@ commands:
|
|||
cd ~/project
|
||||
export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
|
||||
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 tools/stats/upload_binary_size_to_scuba.py android
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba android
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 -mpip install requests && \
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
|
||||
python3 /pytorch/tools/stats/upload_binary_size_to_scuba.py || exit 0
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
- persist_to_workspace:
|
||||
root: /
|
||||
paths: final_pkgs
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ jobs:
|
|||
cd /pytorch && export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
python3 -mpip install requests && \
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN=${SCRIBE_GRAPHQL_ACCESS_TOKEN} \
|
||||
python3 tools/stats/upload_binary_size_to_scuba.py || exit 0
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
- store_artifacts:
|
||||
path: /home/circleci/project/dist
|
||||
|
||||
|
|
|
|||
18
.github/templates/bazel_ci_workflow.yml.j2
vendored
18
.github/templates/bazel_ci_workflow.yml.j2
vendored
|
|
@ -74,6 +74,7 @@ name: Bazel Linux CI (!{{ build_environment }})
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -149,7 +150,7 @@ name: Bazel Linux CI (!{{ build_environment }})
|
|||
if: always()
|
||||
needs:
|
||||
- build-and-test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
steps:
|
||||
- name: Checkout PyTorch
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -165,17 +166,14 @@ name: Bazel Linux CI (!{{ build_environment }})
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -183,9 +181,7 @@ name: Bazel Linux CI (!{{ build_environment }})
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -193,5 +189,5 @@ name: Bazel Linux CI (!{{ build_environment }})
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
{%- endblock %}
|
||||
|
|
|
|||
18
.github/templates/linux_ci_workflow.yml.j2
vendored
18
.github/templates/linux_ci_workflow.yml.j2
vendored
|
|
@ -161,6 +161,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -383,7 +384,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -402,17 +403,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -420,9 +418,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: !{{ build_environment }}-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -430,7 +426,7 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
{%- endblock %}
|
||||
{%- if enable_doc_jobs %}
|
||||
|
||||
|
|
|
|||
17
.github/templates/windows_ci_workflow.yml.j2
vendored
17
.github/templates/windows_ci_workflow.yml.j2
vendored
|
|
@ -216,7 +216,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -235,17 +235,14 @@ jobs:
|
|||
- name: Unzip test reports
|
||||
run: |
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -253,9 +250,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: !{{ build_environment }}-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -263,4 +258,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
2
.github/workflows/build_linux_conda.yml
vendored
2
.github/workflows/build_linux_conda.yml
vendored
|
|
@ -105,7 +105,7 @@ jobs:
|
|||
COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
|
||||
export COMMIT_TIME
|
||||
pip3 install requests
|
||||
python3 tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
|
||||
|
||||
concurrency:
|
||||
group: build-linux-conda-${{ github.event.pull_request.number || github.sha }}
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -367,7 +368,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -386,17 +387,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -404,9 +402,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: periodic-pytorch-linux-xenial-cuda11.3-cudnn8-py3.6-gcc7-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -414,4 +410,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -209,17 +209,14 @@ jobs:
|
|||
- name: Unzip test reports
|
||||
run: |
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -227,9 +224,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: periodic-pytorch-win-vs2019-cuda11-cudnn8-py3-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -237,4 +232,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -369,7 +370,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -388,17 +389,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -406,9 +404,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -416,4 +412,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -370,7 +371,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -389,17 +390,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -407,9 +405,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-bionic-py3.8-gcc9-coverage-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -417,4 +413,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -369,7 +370,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -388,17 +389,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -406,9 +404,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -416,4 +412,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -369,7 +370,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -388,17 +389,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -406,9 +404,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -416,4 +412,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -152,6 +152,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -370,7 +371,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -389,17 +390,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -407,9 +405,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc5.4-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -417,7 +413,7 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
||||
pytorch_python_doc_build:
|
||||
runs-on: linux.2xlarge
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -239,7 +240,7 @@ jobs:
|
|||
if: always()
|
||||
needs:
|
||||
- build-and-test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
steps:
|
||||
- name: Checkout PyTorch
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -255,17 +256,14 @@ jobs:
|
|||
run: |
|
||||
# Should preserve paths so reports should still be in test/test-reports
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -273,9 +271,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-linux-xenial-py3.6-gcc7-bazel-test-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -283,4 +279,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
17
.github/workflows/pytorch-win-vs2019-cpu-py3.yml
vendored
17
.github/workflows/pytorch-win-vs2019-cpu-py3.yml
vendored
|
|
@ -175,7 +175,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -194,17 +194,14 @@ jobs:
|
|||
- name: Unzip test reports
|
||||
run: |
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -212,9 +209,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-win-vs2019-cpu-py3-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -222,4 +217,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -212,17 +212,14 @@ jobs:
|
|||
- name: Unzip test reports
|
||||
run: |
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -230,9 +227,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-win-vs2019-cuda10-cudnn7-py3-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -240,4 +235,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -192,7 +192,7 @@ jobs:
|
|||
needs:
|
||||
- generate-test-matrix
|
||||
- test
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: linux.2xlarge
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.render-matrix) }}
|
||||
fail-fast: false
|
||||
|
|
@ -211,17 +211,14 @@ jobs:
|
|||
- name: Unzip test reports
|
||||
run: |
|
||||
unzip -o 'test-reports-*.zip'
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
# boto3 version copied from .circleci/docker/common/install_conda.sh
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
pip install boto3==1.16.34 junitparser rich
|
||||
pip3 install -r requirements.txt
|
||||
pip3 install boto3==1.16.34 junitparser rich
|
||||
- name: Output Test Results (Click Me)
|
||||
run: |
|
||||
python tools/render_junit.py test
|
||||
python3 tools/render_junit.py test
|
||||
- name: Parse ref
|
||||
id: parse-ref
|
||||
run: .github/scripts/parse_ref.py
|
||||
|
|
@ -229,9 +226,7 @@ jobs:
|
|||
# temporary hack: set CIRCLE_* vars, until we update
|
||||
# tools/stats/print_test_stats.py to natively support GitHub Actions
|
||||
env:
|
||||
SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
|
||||
JOB_BASE_NAME: pytorch-win-vs2019-cuda11-cudnn8-py3-test
|
||||
CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
|
|
@ -239,4 +234,4 @@ jobs:
|
|||
CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
|
||||
CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
|
||||
run: |
|
||||
python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
|
||||
|
|
|
|||
|
|
@ -395,9 +395,9 @@ test_benchmarks() {
|
|||
pytest benchmarks/fastrnns/test_bench.py --benchmark-sort=Name --benchmark-json=${BENCHMARK_DATA}/fastrnns_profiling_te.json --fuser=te --executor=profiling
|
||||
# TODO: Enable these for GHA once we have credentials for forked pull requests
|
||||
if [[ -z "${GITHUB_ACTIONS}" ]]; then
|
||||
python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_default.json
|
||||
python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_legacy_old.json
|
||||
python benchmarks/upload_scribe.py --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_profiling_te.json
|
||||
python -m benchmarks.upload_scribe --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_default.json
|
||||
python -m benchmarks.upload_scribe --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_legacy_old.json
|
||||
python -m benchmarks.upload_scribe --pytest_bench_json ${BENCHMARK_DATA}/fastrnns_profiling_te.json
|
||||
fi
|
||||
assert_git_not_dirty
|
||||
fi
|
||||
|
|
|
|||
0
benchmarks/__init__.py
Normal file
0
benchmarks/__init__.py
Normal file
|
|
@ -10,9 +10,9 @@ import argparse
|
|||
import time
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
from tools.stats.scribe import send_to_scribe
|
||||
|
||||
|
||||
class ScribeUploader:
|
||||
|
|
@ -30,7 +30,6 @@ class ScribeUploader:
|
|||
elif field in self.schema['float']:
|
||||
message['float'][field] = float(value)
|
||||
else:
|
||||
|
||||
raise ValueError("Field {} is not currently used, "
|
||||
"be intentional about adding new fields".format(field))
|
||||
return message
|
||||
|
|
@ -44,28 +43,19 @@ class ScribeUploader:
|
|||
def upload(self, messages):
|
||||
if os.environ.get('SCRIBE_INTERN'):
|
||||
return self._upload_intern(messages)
|
||||
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
|
||||
if not access_token:
|
||||
raise ValueError("Can't find access token from environment variable")
|
||||
url = "https://graph.facebook.com/scribe_logs"
|
||||
r = requests.post(
|
||||
url,
|
||||
data={
|
||||
"access_token": access_token,
|
||||
"logs": json.dumps(
|
||||
[
|
||||
{
|
||||
"category": self.category,
|
||||
"message": json.dumps(message),
|
||||
"line_escape": False,
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
),
|
||||
},
|
||||
logs = json.dumps(
|
||||
[
|
||||
{
|
||||
"category": self.category,
|
||||
"message": json.dumps(message),
|
||||
"line_escape": False,
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
)
|
||||
print(r.text)
|
||||
r.raise_for_status()
|
||||
res = send_to_scribe(logs)
|
||||
print(res)
|
||||
|
||||
|
||||
class PytorchBenchmarkUploader(ScribeUploader):
|
||||
def __init__(self):
|
||||
|
|
|
|||
|
|
@ -16,12 +16,11 @@ from typing import (Any, DefaultDict, Dict, Iterable, Iterator, List, Optional,
|
|||
Set, Tuple, cast)
|
||||
from xml.dom import minidom
|
||||
|
||||
import requests
|
||||
from typing_extensions import TypedDict
|
||||
from tools.stats.s3_stat_parser import (newify_case, get_S3_object_from_bucket, get_test_stats_summaries_for_job,
|
||||
Report, Status, Commit, HAVE_BOTO3, Version2Case, VersionedReport,
|
||||
Version1Report, Version2Report, ReportMetaMeta)
|
||||
|
||||
from tools.stats.scribe import send_to_scribe
|
||||
|
||||
|
||||
SimplerSuite = Dict[str, Version2Case]
|
||||
|
|
@ -610,7 +609,8 @@ class TestSuite:
|
|||
sorted_tests = sorted(self.test_cases.values(), key=lambda x: x.time)
|
||||
test_count = len(sorted_tests)
|
||||
print(f"class {self.name}:")
|
||||
print(f" tests: {test_count} failed: {self.failed_count} skipped: {self.skipped_count} errored: {self.errored_count}")
|
||||
print(
|
||||
f" tests: {test_count} failed: {self.failed_count} skipped: {self.skipped_count} errored: {self.errored_count}")
|
||||
print(f" run_time: {self.total_time:.2f} seconds")
|
||||
print(f" avg_time: {self.total_time/test_count:.2f} seconds")
|
||||
if test_count >= 2:
|
||||
|
|
@ -645,7 +645,8 @@ class TestFile:
|
|||
self.test_suites[suite_name].update(test_case)
|
||||
self.total_time += test_case.time
|
||||
else:
|
||||
raise RuntimeWarning(f'Duplicate test case {test_case.name} in suite {suite_name} called from {self.name}')
|
||||
raise RuntimeWarning(
|
||||
f'Duplicate test case {test_case.name} in suite {suite_name} called from {self.name}')
|
||||
else:
|
||||
self.test_suites[suite_name].append(test_case)
|
||||
self.total_time += test_case.time
|
||||
|
|
@ -729,33 +730,21 @@ def build_message(
|
|||
|
||||
|
||||
def send_report_to_scribe(reports: Dict[str, TestFile]) -> None:
|
||||
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
|
||||
|
||||
if not access_token:
|
||||
print("No scribe access token provided, skip sending report!")
|
||||
return
|
||||
print("Scribe access token provided, sending report...")
|
||||
url = "https://graph.facebook.com/scribe_logs"
|
||||
meta_info = build_info()
|
||||
r = requests.post(
|
||||
url,
|
||||
data={
|
||||
"access_token": access_token,
|
||||
"logs": json.dumps(
|
||||
[
|
||||
{
|
||||
"category": "perfpipe_pytorch_test_times",
|
||||
"message": json.dumps(build_message(test_file, test_suite, test_case, meta_info)),
|
||||
"line_escape": False,
|
||||
}
|
||||
for test_file in reports.values()
|
||||
for test_suite in test_file.test_suites.values()
|
||||
for test_case in test_suite.test_cases.values()
|
||||
]
|
||||
),
|
||||
},
|
||||
logs = json.dumps(
|
||||
[
|
||||
{
|
||||
"category": "perfpipe_pytorch_test_times",
|
||||
"message": json.dumps(build_message(test_file, test_suite, test_case, meta_info)),
|
||||
"line_escape": False,
|
||||
}
|
||||
for test_file in reports.values()
|
||||
for test_suite in test_file.test_suites.values()
|
||||
for test_case in test_suite.test_cases.values()
|
||||
]
|
||||
)
|
||||
r.raise_for_status()
|
||||
res = send_to_scribe(logs)
|
||||
print(res)
|
||||
|
||||
|
||||
def assemble_s3_object(
|
||||
|
|
@ -767,7 +756,7 @@ def assemble_s3_object(
|
|||
**build_info(), # type: ignore[misc]
|
||||
'total_seconds': total_seconds,
|
||||
'format_version': 2,
|
||||
'files' : {
|
||||
'files': {
|
||||
name: {
|
||||
'total_seconds': test_file.total_time,
|
||||
'suites': {
|
||||
|
|
@ -880,6 +869,7 @@ def reports_has_no_tests(reports: Dict[str, TestFile]) -> bool:
|
|||
return False
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
import sys
|
||||
|
|
@ -948,7 +938,7 @@ if __name__ == '__main__':
|
|||
print(f"ERROR ENCOUNTERED WHEN UPLOADING TO SCRIBE: {e}")
|
||||
|
||||
# longest_tests can contain duplicates as the same tests can be spawned from different files
|
||||
longest_tests : List[TestCase] = []
|
||||
longest_tests: List[TestCase] = []
|
||||
total_time = 0.0
|
||||
for filename, test_filename in reports_by_file.items():
|
||||
for suite_name, test_suite in test_filename.test_suites.items():
|
||||
|
|
|
|||
42
tools/stats/scribe.py
Normal file
42
tools/stats/scribe.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import base64
|
||||
import bz2
|
||||
import os
|
||||
import json
|
||||
|
||||
|
||||
def send_to_scribe(logs: str) -> str:
|
||||
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN", "")
|
||||
|
||||
# boto3 can be used when the runner has IAM roles setup
|
||||
# currently it's used as a fallback when SCRIBE_GRAPHQL_ACCESS_TOKEN is empty
|
||||
if access_token == "":
|
||||
return _send_to_scribe_via_boto3(logs)
|
||||
|
||||
return _send_to_scribe_via_http(access_token, logs)
|
||||
|
||||
|
||||
def _send_to_scribe_via_boto3(logs: str) -> str:
|
||||
# lazy import so that we don't need to introduce extra dependencies
|
||||
import boto3 # type: ignore[import]
|
||||
|
||||
print("Scribe access token not provided, sending report via boto3...")
|
||||
event = {"base64_bz2_logs": base64.b64encode(bz2.compress(logs.encode())).decode()}
|
||||
client = boto3.client("lambda")
|
||||
res = client.invoke(FunctionName='gh-ci-scribe-proxy', Payload=json.dumps(event).encode())
|
||||
payload = str(res['Payload'].read().decode())
|
||||
if res.get('FunctionError'):
|
||||
raise Exception(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _send_to_scribe_via_http(access_token: str, logs: str) -> str:
|
||||
# lazy import so that we don't need to introduce extra dependencies
|
||||
import requests # type: ignore[import]
|
||||
|
||||
print("Scribe access token provided, sending report via http...")
|
||||
r = requests.post(
|
||||
"https://graph.facebook.com/scribe_logs",
|
||||
data={"access_token": access_token, "logs": logs},
|
||||
)
|
||||
r.raise_for_status()
|
||||
return str(r.text)
|
||||
|
|
@ -9,9 +9,8 @@ import sys
|
|||
import time
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
|
||||
from typing import Any, Dict, Generator, List
|
||||
from tools.stats.scribe import send_to_scribe
|
||||
|
||||
|
||||
def get_size(file_dir: str) -> int:
|
||||
|
|
@ -52,28 +51,18 @@ def build_message(size: int) -> Dict[str, Any]:
|
|||
|
||||
|
||||
def send_message(messages: List[Dict[str, Any]]) -> None:
|
||||
access_token = os.environ.get("SCRIBE_GRAPHQL_ACCESS_TOKEN")
|
||||
if not access_token:
|
||||
raise ValueError("Can't find access token from environment variable")
|
||||
url = "https://graph.facebook.com/scribe_logs"
|
||||
r = requests.post(
|
||||
url,
|
||||
data={
|
||||
"access_token": access_token,
|
||||
"logs": json.dumps(
|
||||
[
|
||||
{
|
||||
"category": "perfpipe_pytorch_binary_size",
|
||||
"message": json.dumps(message),
|
||||
"line_escape": False,
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
),
|
||||
},
|
||||
logs = json.dumps(
|
||||
[
|
||||
{
|
||||
"category": "perfpipe_pytorch_binary_size",
|
||||
"message": json.dumps(message),
|
||||
"line_escape": False,
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
)
|
||||
print(r.text)
|
||||
r.raise_for_status()
|
||||
res = send_to_scribe(logs)
|
||||
print(res)
|
||||
|
||||
|
||||
def report_android_sizes(file_dir: str) -> None:
|
||||
|
|
|
|||
Loading…
Reference in a new issue