From 7081923cb759dd84d7675651ac41d69e161e306e Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Sun, 16 Aug 2020 17:50:38 -0500 Subject: [PATCH] refactor wheel builder to use workflows and download/build separately (#5412) * refactor wheel builder to use workflows and download/build separately * Update .github/workflows/wheel-builder.yml Co-authored-by: Alex Gaynor Co-authored-by: Alex Gaynor --- .github/workflows/wheel-builder.yml | 68 +++++++++++++++-------------- release.py | 14 +++--- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/.github/workflows/wheel-builder.yml b/.github/workflows/wheel-builder.yml index d57e2a832..d8ad49b59 100644 --- a/.github/workflows/wheel-builder.yml +++ b/.github/workflows/wheel-builder.yml @@ -1,7 +1,9 @@ name: Wheel Builder on: - repository_dispatch: - types: [wheel-builder] + workflow_dispatch: + inputs: + version: + required: true jobs: manylinux: @@ -15,21 +17,21 @@ jobs: CONTAINER: "pyca/cryptography-manylinux1:x86_64" - NAME: manylinux2010_x86_64 CONTAINER: "pyca/cryptography-manylinux2010:x86_64" - name: "Python ${{ matrix.PYTHON }} for ${{ matrix.MANYLINUX.NAME }}" + name: "${{ matrix.PYTHON }} for ${{ matrix.MANYLINUX.NAME }}" steps: - run: /opt/python/${{ matrix.PYTHON }}/bin/python -m virtualenv .venv - - name: Downgrade pip, can't remember why - run: .venv/bin/pip install -U pip==10.0.1 - name: Install Python dependencies - run: .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'" + run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'" + - run: .venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir tmpwheelhouse - run: | REGEX="cp3([0-9])*" if [[ "${{ matrix.PYTHON }}" =~ $REGEX ]]; then - PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}" + PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" fi + cd cryptography* LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \ CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \ - .venv/bin/pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --no-binary cryptography --no-deps --wheel-dir=tmpwheelhouse $PY_LIMITED_API + ../.venv/bin/python setup.py bdist_wheel $PY_LIMITED_API && mv dist/cryptography*.whl ../tmpwheelhouse - run: auditwheel repair --plat ${{ matrix.MANYLINUX.NAME }} tmpwheelhouse/cryptograph*.whl -w wheelhouse/ - run: unzip wheelhouse/*.whl -d execstack.check - run: | @@ -40,8 +42,6 @@ jobs: else exit 0 fi - - name: Upgrade pip again so we can actually use manylinux2010 - run: .venv/bin/pip install -U pip - run: .venv/bin/pip install cryptography --no-index -f wheelhouse/ - run: | .venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" @@ -49,7 +49,7 @@ jobs: - run: mv wheelhouse/cryptography*.whl cryptography-wheelhouse/ - uses: actions/upload-artifact@v1 with: - name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.MANYLINUX.NAME }}-${{ matrix.PYTHON }}" + name: "cryptography-${{ github.event.inputs.version }}-${{ matrix.MANYLINUX.NAME }}-${{ matrix.PYTHON }}" path: cryptography-wheelhouse/ macos: @@ -65,7 +65,7 @@ jobs: ABI_VERSION: '3.5' DOWNLOAD_URL: 'https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg' BIN_PATH: '/Library/Frameworks/Python.framework/Versions/3.8/bin/python3' - name: "Python ${{ matrix.PYTHON.VERSION }} for ABI ${{ matrix.PYTHON.ABI_VERSION }} on macOS" + name: "${{ matrix.PYTHON.VERSION }} ABI ${{ matrix.PYTHON.ABI_VERSION }} macOS" steps: - uses: actions/checkout@master - run: | @@ -81,24 +81,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ${{ matrix.PYTHON.BIN_PATH }} -m virtualenv venv - # Downgrade pip, I can't remember why - - run: venv/bin/pip install -U pip==10.0.1 - - run: venv/bin/pip install -U wheel - - run: venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'" - + - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'" + - run: venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse - name: Build the wheel run: | REGEX="3\.([0-9])*" - if [[ "$PYTHON_VERSION" =~ $REGEX ]]; then - PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}" + if [[ "${{ matrix.PYTHON.ABI_VERSION }}" =~ $REGEX ]]; then + PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" fi + cd cryptography* CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS="1" \ LDFLAGS="${HOME}/openssl-macos/lib/libcrypto.a ${HOME}/openssl-macos/lib/libssl.a" \ CFLAGS="-I${HOME}/openssl-macos/include -mmacosx-version-min=10.10 -march=core2" \ - venv/bin/pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary cryptography --no-deps $PY_LIMITED_API - env: - PYTHON_VERSION: ${{ matrix.PYTHON.ABI_VERSION }} + ../venv/bin/python setup.py bdist_wheel $PY_LIMITED_API && mv dist/cryptography*.whl ../wheelhouse - run: venv/bin/pip install -f wheelhouse --no-index cryptography - run: | venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" @@ -107,7 +103,7 @@ jobs: - run: mv wheelhouse/cryptography*.whl cryptography-wheelhouse/ - uses: actions/upload-artifact@v1 with: - name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-macOS-${{ matrix.PYTHON.ABI_VERSION }}" + name: "cryptography-${{ github.event.inputs.version }}-macOS-${{ matrix.PYTHON.ABI_VERSION }}" path: cryptography-wheelhouse/ windows: @@ -118,16 +114,17 @@ jobs: - {ARCH: 'x86', WINDOWS: 'win32'} - {ARCH: 'x64', WINDOWS: 'win64'} PYTHON: - - {VERSION: "2.7", TOXENV: "py27", MSVC_VERSION: "2010"} - - {VERSION: "3.5", TOXENV: "py35", MSVC_VERSION: "2019"} - - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019"} - - {VERSION: "3.7", TOXENV: "py37", MSVC_VERSION: "2019"} - - {VERSION: "3.8", TOXENV: "py38", MSVC_VERSION: "2019"} - name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}" + - {VERSION: "2.7", MSVC_VERSION: "2010"} + - {VERSION: "3.5", MSVC_VERSION: "2019"} + - {VERSION: "3.6", MSVC_VERSION: "2019"} + - {VERSION: "3.7", MSVC_VERSION: "2019"} + - {VERSION: "3.8", MSVC_VERSION: "2019"} + - {VERSION: "3.8", MSVC_VERSION: "2019", "USE_ABI3": "true", "ABI_VERSION": "cp36"} + name: "${{ matrix.PYTHON.VERSION }} ${{ matrix.WINDOWS.WINDOWS }} ${{ matrix.PYTHON.ABI_VERSION }}" steps: - uses: actions/checkout@master - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.PYTHON.VERSION }} architecture: ${{ matrix.WINDOWS.ARCH }} @@ -147,8 +144,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: pip install cffi six ipaddress "enum34; python_version < '3'" - - run: pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary cryptography + - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'" + - run: pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse + shell: bash + - run: cd cryptography* && python setup.py bdist_wheel && mv dist/cryptography*.whl ../wheelhouse + if: matrix.PYTHON.USE_ABI3 != 'true' + - run: cd cryptography* && python setup.py bdist_wheel --py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} && mv dist/cryptography*.whl ../wheelhouse + if: matrix.PYTHON.USE_ABI3 == 'true' - run: pip install -f wheelhouse --no-index cryptography - name: Print the OpenSSL we built and linked against run: | @@ -158,5 +160,5 @@ jobs: - run: move wheelhouse\cryptography*.whl cryptography-wheelhouse\ - uses: actions/upload-artifact@v1 with: - name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}" + name: "cryptography-${{ github.event.inputs.version }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}-${{ matrix.PYTHON.ABI_VERSION}}" path: cryptography-wheelhouse\ diff --git a/release.py b/release.py index a5eea848e..550c261dc 100644 --- a/release.py +++ b/release.py @@ -83,18 +83,14 @@ def build_github_actions_wheels(token, version): session = requests.Session() response = session.post( - "https://api.github.com/repos/pyca/cryptography/dispatches", + "https://api.github.com/repos/pyca/cryptography/actions/workflows/" + "wheel-builder.yml/dispatches", headers={ "Content-Type": "application/json", - "Accept": "application/vnd.github.everest-preview+json", + "Accept": "application/vnd.github.v3+json", "Authorization": "token {}".format(token), }, - data=json.dumps( - { - "event_type": "wheel-builder", - "client_payload": {"BUILD_VERSION": version}, - } - ), + data=json.dumps({"ref": "master", "inputs": {"version": version}}), ) response.raise_for_status() @@ -103,7 +99,7 @@ def build_github_actions_wheels(token, version): response = session.get( ( "https://api.github.com/repos/pyca/cryptography/actions/workflows/" - "wheel-builder.yml/runs?event=repository_dispatch" + "wheel-builder.yml/runs?event=workflow_dispatch" ), headers={ "Content-Type": "application/json",