pytorch/.github/workflows/_binary-upload.yml

name: upload

on:
  workflow_call:
    inputs:
      build_name:
        required: true
        type: string
        description: The build's name
      use_s3:
        type: boolean
        default: true
        description: If true, will download artifacts from s3. Otherwise will use the default github artifact download action
      PYTORCH_ROOT:
        required: false
        type: string
        description: Root directory for the pytorch/pytorch repository. Not actually needed, but currently passing it in since we pass in the same inputs to the reusable workflows of all binary builds
      BUILDER_ROOT:
        required: false
        type: string
        description: Root directory for the pytorch/builder repository. Not actually needed, but currently passing it in since we pass in the same inputs to the reusable workflows of all binary builds
      PACKAGE_TYPE:
        required: true
        type: string
        description: Package type
      DESIRED_CUDA:
        required: true
        type: string
        description: Desired Cuda version
      GPU_ARCH_VERSION:
        required: false
        type: string
        description: GPU Arch version
      GPU_ARCH_TYPE:
        required: true
        type: string
        description: GPU Arch type
      DOCKER_IMAGE:
        required: false
        type: string
        description: Docker image to use
      LIBTORCH_CONFIG:
        required: false
        type: string
        description: Desired libtorch config (for libtorch builds only)
      LIBTORCH_VARIANT:
        required: false
        type: string
        description: Desired libtorch variant (for libtorch builds only)
      DESIRED_DEVTOOLSET:
        required: false
        type: string
        description: Desired dev toolset
      DESIRED_PYTHON:
        required: false
        type: string
        description: Desired python version
    secrets:
      github-token:
        required: true
        description: Github Token
      aws-access-key-id:
        required: true
        description: AWS access key id
      aws-pytorch-uploader-secret-access-key:
        required: true
        description: AWS secret access key
      conda-pytorchbot-token:
        required: true
        description: Conda PyTorchBot token
jobs:
  build:
    runs-on: linux.2xlarge
    env:
      PYTORCH_ROOT: /pytorch
      BUILDER_ROOT: /builder
      PACKAGE_TYPE: ${{ inputs.PACKAGE_TYPE }}
      # TODO: This is a legacy variable that we eventually want to get rid of in
      #       favor of GPU_ARCH_VERSION
      DESIRED_CUDA: ${{ inputs.DESIRED_CUDA }}
      GPU_ARCH_VERSION: ${{ inputs.GPU_ARCH_VERSION }}
      GPU_ARCH_TYPE: ${{ inputs.GPU_ARCH_TYPE }}
      DOCKER_IMAGE: ${{ inputs.DOCKER_IMAGE }}
      SKIP_ALL_TESTS: 1
      LIBTORCH_CONFIG: ${{ inputs.LIBTORCH_CONFIG }}
      LIBTORCH_VARIANT: ${{ inputs.LIBTORCH_VARIANT }}
      DESIRED_DEVTOOLSET: ${{ inputs.DESIRED_DEVTOOLSET }}
      DESIRED_PYTHON: ${{ inputs.DESIRED_PYTHON }}
      # Needed for conda builds
      ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
      ANACONDA_USER: pytorch
      AWS_DEFAULT_REGION: us-east-1
      BINARY_ENV_FILE: /tmp/env
      GITHUB_TOKEN: ${{ secrets.github-token }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
      PYTORCH_FINAL_PACKAGE_DIR: /artifacts
      SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
    steps:
      - name: List the env
        shell: bash
        run: env
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
      - name: Setup Linux
        uses: ./.github/actions/setup-linux
      - name: Chown workspace
        uses: ./.github/actions/chown-workspace
      - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
        uses: ./.github/actions/setup-ssh
        with:
          github-secret: ${{ secrets.github-token }}

      - name: Download Build Artifacts with S3
        uses: seemethere/download-artifact-s3@v4
        if: ${{ inputs.use_s3 }}
        with:
          name: ${{ inputs.build_name }}
          path: "${{ runner.temp }}/artifacts/"

      - name: Download Build Artifacts without S3
        uses: actions/download-artifact@v2
        if: ${{ !inputs.use_s3 }}
        with:
          name: ${{ inputs.build_name }}
          path: "${{ runner.temp }}/artifacts/"

      - name: Set DRY_RUN (only for tagged pushes)
        if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/nightly' || (startsWith(github.event.ref, 'refs/tags/') && !startsWith(github.event.ref, 'refs/tags/ciflow/'))) }}
        run: |
          echo "DRY_RUN=disabled" >> "$GITHUB_ENV"          
      - name: Set UPLOAD_CHANNEL (only for tagged pushes)
        if: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') && !startsWith(github.event.ref, 'refs/tags/ciflow/') }}
        run: |
          # reference ends with an RC suffix
          if [[ ${GITHUB_REF_NAME} = *-rc[0-9]* ]]; then
            echo "UPLOAD_CHANNEL=test" >> "$GITHUB_ENV"
          fi          
      - name: Upload binaries
        env:
          PKG_DIR: "${{ runner.temp }}/artifacts"
          UPLOAD_SUBFOLDER: "${{ env.DESIRED_CUDA }}"
          # When running these on pull_request events these should be blank
          AWS_ACCESS_KEY_ID: ${{ secrets.aws-access-key-id }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.aws-pytorch-uploader-secret-access-key }}
          ANACONDA_API_TOKEN: ${{ secrets.conda-pytorchbot-token }}
        run: |
          docker run --rm -i \
            -e ANACONDA_API_TOKEN \
            -e AWS_ACCESS_KEY_ID \
            -e AWS_SECRET_ACCESS_KEY \
            -e DRY_RUN \
            -e PACKAGE_TYPE \
            -e PKG_DIR=/artifacts \
            -e UPLOAD_CHANNEL \
            -e UPLOAD_SUBFOLDER \
            -v "${RUNNER_TEMP}/artifacts:/artifacts" \
            -v "${GITHUB_WORKSPACE}:/v" \
            -w /v \
            308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/miniconda3:4.10.3 \
            bash -c '.circleci/scripts/binary_upload.sh'          

      - name: Hold runner for 2 hours or until ssh sessions have drained
        # Always hold for active ssh sessions
        if: always()
        run: .github/scripts/wait_for_ssh_to_drain.sh
      - name: Chown workspace
        if: always()
        run: |
          # Ensure the working directory gets chowned back to the current user
          docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .          
      - name: Kill containers, clean up images
        if: always()
        run: |
          # ignore expansion of "docker ps -q" since it could be empty
          # shellcheck disable=SC2046
          docker stop $(docker ps -q) || true
          # Prune all of the docker images
          docker system prune -af