mirror of
https://github.com/saymrwulf/pytorch.git
synced 2026-05-15 21:00:47 +00:00
04c1c2b791
There is a bug in the test workflow where it could fail to find the new Docker image when the image hasn't yet became available on ECR, for example e71ab21422. This basically is a race condition where the test job starts before the docker-build workflow could finish successfully. The fix here is to make sure that the test job has the opportunity to build the image if it doesn't exist, same as what the build workflow does atm. Once the docker-build workflow finishes pushing the new image to ECR, that can then be used instead.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/102562
Approved by: https://github.com/PaliC
187 lines
7.4 KiB
YAML
187 lines
7.4 KiB
YAML
name: build docs
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
build-environment:
|
|
required: true
|
|
type: string
|
|
description: Top-level label for what's being built/tested.
|
|
docker-image:
|
|
required: true
|
|
type: string
|
|
description: Docker image to run in.
|
|
push:
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
description: If set, push the docs to the docs website.
|
|
run-doxygen:
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
description: If set, will enable C++ API doc generation using doxygen / breathe / exhale.
|
|
sync-tag:
|
|
required: false
|
|
type: string
|
|
default: ""
|
|
description: |
|
|
If this is set, our linter will use this to make sure that every other
|
|
job with the same `sync-tag` is identical.
|
|
|
|
secrets:
|
|
GH_PYTORCHBOT_TOKEN:
|
|
required: false
|
|
description: Permissions for pushing to the docs site.
|
|
|
|
jobs:
|
|
build-docs:
|
|
# Don't run on forked repos.
|
|
if: github.repository_owner == 'pytorch'
|
|
runs-on: ${{ matrix.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- docs_type: cpp
|
|
# We recently seeing lots of exit code 137 running this in Docker indicating
|
|
# an OOM issue when running the job, so this upgrades the runner from 4xlarge
|
|
# to the next available tier of 12xlarge. So much memory just to generate cpp
|
|
# doc
|
|
runner: linux.12xlarge
|
|
# TODO: Nightly cpp docs take longer and longer to finish (more than 3h now)
|
|
# Let's try to figure out how this can be improved
|
|
timeout-minutes: 240
|
|
- docs_type: python
|
|
runner: linux.2xlarge
|
|
# It takes less than 30m to finish python docs unless there are issues
|
|
timeout-minutes: 30
|
|
- docs_type: functorch
|
|
runner: linux.2xlarge
|
|
# It takes less than 15m to finish functorch docs unless there are issues
|
|
timeout-minutes: 15
|
|
# Set a fixed name for this job instead of using the current matrix-generated name, i.e. build-docs (cpp, linux.12xlarge, 180)
|
|
# The current name requires updating the Rockset last docs push query from test-infra every time the matrix is updated
|
|
name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }}
|
|
steps:
|
|
- name: Setup SSH (Click me for login details)
|
|
uses: pytorch/test-infra/.github/actions/setup-ssh@main
|
|
with:
|
|
github-secret: ${{ secrets.GITHUB_TOKEN }}
|
|
instructions: |
|
|
All builds are done inside the container, to start an interactive session run:
|
|
docker exec -it $(docker container ps --format '{{.ID}}') bash
|
|
To start Python docs build type:
|
|
cd docs && make html && make coverage
|
|
|
|
# [see note: pytorch repo ref]
|
|
- name: Checkout PyTorch
|
|
uses: pytorch/pytorch/.github/actions/checkout-pytorch@main
|
|
|
|
- name: Setup Linux
|
|
uses: ./.github/actions/setup-linux
|
|
|
|
- name: Calculate docker image
|
|
id: calculate-docker-image
|
|
uses: ./.github/actions/calculate-docker-image
|
|
with:
|
|
docker-image-name: ${{ inputs.docker-image }}
|
|
xla: ${{ contains(inputs.build-environment, 'xla') }}
|
|
|
|
- name: Pull docker image
|
|
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
|
|
with:
|
|
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
|
|
|
|
- name: Download build artifacts
|
|
uses: ./.github/actions/download-build-artifacts
|
|
with:
|
|
name: ${{ inputs.build-environment }}
|
|
|
|
- name: Generate netrc (only for docs-push)
|
|
if: inputs.push
|
|
env:
|
|
GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
|
|
run: |
|
|
# sometimes .netrc exists as a directory even though this is the temp folder
|
|
rm -rf "${RUNNER_TEMP}/.netrc"
|
|
# set credentials for https pushing
|
|
echo "machine github.com" > "${RUNNER_TEMP}/.netrc"
|
|
echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc"
|
|
echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc"
|
|
|
|
- name: Build ${{ matrix.docs_type }} docs
|
|
timeout-minutes: ${{ matrix.timeout-minutes }}
|
|
id: build-docs
|
|
env:
|
|
# After https://github.com/pytorch/pytorch/pull/88373, pull workflow can now be run periodically,
|
|
# so using a schedule event to determine if the docs should be pushed or not doesn't hold true
|
|
# anymore
|
|
WITH_PUSH: ${{ inputs.push }}
|
|
DOCKER_IMAGE: ${{ inputs.docker-image }}
|
|
DOCS_TYPE: ${{ matrix.docs_type }}
|
|
RUN_DOXYGEN: ${{ inputs.run-doxygen }}
|
|
BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
|
|
run: |
|
|
set -ex
|
|
# Convert refs/tags/v1.12.0rc3 into 1.12
|
|
if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then
|
|
target="${BASH_REMATCH[1]}"
|
|
else
|
|
target="main"
|
|
fi
|
|
# detached container should get cleaned up by teardown_ec2_linux
|
|
container_name=$(docker run \
|
|
-e BUILD_ENVIRONMENT \
|
|
-e MAX_JOBS="$(nproc --ignore=2)" \
|
|
-e SHA1="$GITHUB_SHA" \
|
|
-e DOCS_VERSION="${target}" \
|
|
-e DOCS_TYPE \
|
|
-e RUN_DOXYGEN \
|
|
-e WITH_PUSH \
|
|
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
|
|
--security-opt seccomp=unconfined \
|
|
--cap-add=SYS_PTRACE \
|
|
--tty \
|
|
--detach \
|
|
--user jenkins \
|
|
-v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \
|
|
-v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
|
|
-w /var/lib/jenkins/workspace \
|
|
"${DOCKER_IMAGE}"
|
|
)
|
|
docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install $(echo dist/*.whl)[opt-einsum] && ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh"
|
|
|
|
- name: Chown workspace
|
|
uses: ./.github/actions/chown-workspace
|
|
if: always()
|
|
|
|
- name: Upload Python Docs Preview
|
|
uses: seemethere/upload-artifact-s3@v5
|
|
if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }}
|
|
with:
|
|
retention-days: 14
|
|
s3-bucket: doc-previews
|
|
if-no-files-found: error
|
|
path: pytorch.github.io/docs/main/
|
|
s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}
|
|
|
|
- name: Upload C++ Docs Preview
|
|
uses: seemethere/upload-artifact-s3@v5
|
|
if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
|
|
with:
|
|
retention-days: 14
|
|
if-no-files-found: error
|
|
s3-bucket: doc-previews
|
|
path: cppdocs/
|
|
s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs
|
|
|
|
- name: Upload functorch Docs Preview
|
|
uses: seemethere/upload-artifact-s3@v5
|
|
if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'functorch' && steps.build-docs.outcome == 'success' }}
|
|
with:
|
|
retention-days: 14
|
|
s3-bucket: doc-previews
|
|
if-no-files-found: error
|
|
path: functorch_ghpages/nightly/
|
|
s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/functorchdocs
|