From 1685167e46782ac4a61066f44d7a1ca8d875e9c2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 31 Dec 2020 19:57:04 -0800 Subject: [PATCH] Update manylinux docker image to the latest (#6242) --- cgmanifests/cgmanifest.json | 10 +-- cgmanifests/submodules/cgmanifest.json | 43 ++++++++-- .../generate_submodule_cgmanifest.py | 80 +++++++++++-------- .../manylinux2014_build_scripts/build.sh | 10 +-- .../manylinux2014_build_scripts/build_env.sh | 13 +-- .../build_utils.sh | 23 +++--- .../requirements-tools.txt | 16 ++-- .../requirements.txt | 17 ++-- 8 files changed, 131 insertions(+), 81 deletions(-) diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index 3850e8c909..fde4e5ef97 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -343,7 +343,7 @@ "component": { "type": "git", "git": { - "commitHash": "bc8ce45b35ade9b8f08d88d4d9fdfd12b4f6f310", + "commitHash": "56195b3a67d988fa9c6eafdc459d92f0909f4782", "repositoryUrl": "https://github.com/pypa/manylinux" }, "comments": "For building our CI build docker image" @@ -363,20 +363,20 @@ "component": { "type": "git", "git": { - "commitHash": "9cf6752276e6fcfd0c23fdb064ad27f448aaaf75", + "commitHash": "1e5d33e9b9b8631b36f061103a30208b206fd03a", "repositoryUrl": "https://github.com/python/cpython" }, - "comments": "Python 3.9.0" + "comments": "Python 3.9.1" } }, { "component": { "type": "git", "git": { - "commitHash": "db455296be5f792b8c12b7cd7f3962b52e4f44ee", + "commitHash": "6503f05dd59e26a9986bdea097b3da9b3546f45b", "repositoryUrl": "https://github.com/python/cpython" }, - "comments": "Python 3.8.6" + "comments": "Python 3.8.7" } }, { diff --git a/cgmanifests/submodules/cgmanifest.json b/cgmanifests/submodules/cgmanifest.json index dfb31ed6c1..15672d4c86 100644 --- a/cgmanifests/submodules/cgmanifest.json +++ b/cgmanifests/submodules/cgmanifest.json @@ -1,6 +1,17 @@ { "Version": 1, "Registrations": [ + { + "Component": { + "Type": "other", + "other": { + "Name": "patchelf", + "Version": "0.12", + "DownloadUrl": "https://github.com/NixOS/patchelf/archive/0.12.tar.gz" + }, + "comments": "manylinux dependency" + } + }, { "Component": { "Type": "other", @@ -27,9 +38,31 @@ "Component": { "Type": "other", "other": { - "Name": "sqlite", - "Version": "3330000", - "DownloadUrl": "https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz" + "Name": "sqlite_autoconf", + "Version": "3340000", + "DownloadUrl": "https://www.sqlite.org/2020/sqlite-autoconf-3340000.tar.gz" + }, + "comments": "manylinux dependency" + } + }, + { + "Component": { + "Type": "other", + "other": { + "Name": "cmake", + "Version": "3.18.3", + "DownloadUrl": "https://github.com/Kitware/CMake/releases/download/v3.18.3/cmake-3.18.3.tar.gz" + }, + "comments": "manylinux dependency" + } + }, + { + "Component": { + "Type": "other", + "other": { + "Name": "libxcrypt", + "Version": "4.4.17", + "DownloadUrl": "https://github.com/besser82/libxcrypt/archive/v4.4.17.tar.gz" }, "comments": "manylinux dependency" } @@ -39,8 +72,8 @@ "Type": "other", "other": { "Name": "git", - "Version": "2.29.1", - "DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.29.1.tar.gz" + "Version": "2.30.0", + "DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.30.0.tar.gz" }, "comments": "manylinux dependency" } diff --git a/cgmanifests/submodules/generate_submodule_cgmanifest.py b/cgmanifests/submodules/generate_submodule_cgmanifest.py index a1ee7d909e..2f089e27d8 100644 --- a/cgmanifests/submodules/generate_submodule_cgmanifest.py +++ b/cgmanifests/submodules/generate_submodule_cgmanifest.py @@ -15,41 +15,51 @@ package_url = None registrations = [] -with open(os.path.join(REPO_DIR,'tools','ci_build','github','linux','docker','manylinux2014_build_scripts','build_env.sh'), "r") as f: - for line in f: - if not line.strip(): - package_name = None - package_filename = None - package_url = None - if package_filename is None: - m = re.match("(.+?)_ROOT=(.*)$",line) - if m != None: - package_name = m.group(1) - package_filename = m.group(2) - else: - m = re.match("(.+?)_AUTOCONF_VERSION=(.*)$",line) - if m != None: - package_name = m.group(1) - package_filename = m.group(2) - elif package_url is None: - m = re.match("(.+?)_DOWNLOAD_URL=(.+)$",line) - if m != None: - package_url = m.group(2) + "/" + package_filename + ".tar.gz" - registration = { - "Component": { - "Type": "other", - "other": { - "Name": package_name.lower(), - "Version": package_filename.split("-")[-1], - "DownloadUrl": package_url, - }, - "comments": "manylinux dependency" - } - } - registrations.append(registration) - package_name = None - package_filename = None - package_url = None +with open(os.path.join(REPO_DIR, 'tools', 'ci_build', 'github', 'linux', 'docker', 'manylinux2014_build_scripts', + 'build_env.sh'), "r") as f: + for line in f: + if not line.strip(): + package_name = None + package_filename = None + package_url = None + if package_filename is None: + m = re.match("(.+?)_ROOT=(.*)$", line) + if m is not None: + package_name = m.group(1) + package_filename = m.group(2) + else: + m = re.match("(.+?)_VERSION=(.*)$", line) + if m is not None: + package_name = m.group(1) + package_filename = m.group(2) + elif package_url is None: + m = re.match("(.+?)_DOWNLOAD_URL=(.+)$", line) + if m is not None: + package_url = m.group(2) + if package_name == 'LIBXCRYPT': + package_url = m.group(2) + "/v" + \ + package_filename + ".tar.gz" + elif package_name == 'CMAKE': + package_url = m.group( + 2) + "/v" + package_filename + "/cmake-" + package_filename + ".tar.gz" + else: + package_url = m.group(2) + "/" + \ + package_filename + ".tar.gz" + registration = { + "Component": { + "Type": "other", + "other": { + "Name": package_name.lower(), + "Version": package_filename.split("-")[-1], + "DownloadUrl": package_url, + }, + "comments": "manylinux dependency" + } + } + registrations.append(registration) + package_name = None + package_filename = None + package_url = None proc = subprocess.run( diff --git a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build.sh b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build.sh index 816a0cdf91..df7646b779 100755 --- a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build.sh +++ b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build.sh @@ -111,13 +111,13 @@ build_libtool $LIBTOOL_ROOT $LIBTOOL_HASH libtool --version # Install a more recent SQLite3 -curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_VERSION.tar.gz -check_sha256sum $SQLITE_AUTOCONF_VERSION.tar.gz $SQLITE_AUTOCONF_HASH -tar xfz $SQLITE_AUTOCONF_VERSION.tar.gz -cd $SQLITE_AUTOCONF_VERSION +curl -fsSLO $SQLITE_AUTOCONF_DOWNLOAD_URL/$SQLITE_AUTOCONF_ROOT.tar.gz +check_sha256sum $SQLITE_AUTOCONF_ROOT.tar.gz $SQLITE_AUTOCONF_HASH +tar xfz $SQLITE_AUTOCONF_ROOT.tar.gz +cd $SQLITE_AUTOCONF_ROOT do_standard_install cd .. -rm -rf $SQLITE_AUTOCONF_VERSION* +rm -rf $SQLITE_AUTOCONF_ROOT* rm /usr/local/lib/libsqlite3.a # Install a recent version of cmake3 diff --git a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_env.sh b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_env.sh index 4b8a18715e..3daa21d7f3 100644 --- a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_env.sh +++ b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_env.sh @@ -2,10 +2,11 @@ PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python # of the form .. or ..rc -CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.6 3.9.0" +CPYTHON_VERSIONS="3.5.10 3.6.12 3.7.9 3.8.7 3.9.1" PATCHELF_VERSION=0.12 PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 +PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive AUTOMAKE_ROOT=automake-1.16.2 AUTOMAKE_HASH=b2f361094b410b4acbf4efba7337bdb786335ca09eb2518635a09fb7319ca5c1 @@ -15,8 +16,8 @@ LIBTOOL_ROOT=libtool-2.4.6 LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool -SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3330000 -SQLITE_AUTOCONF_HASH=106a2c48c7f75a298a7557bcc0d5f4f454e5b43811cc738b7ca294d6956bbb15 +SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3340000 +SQLITE_AUTOCONF_HASH=bf6db7fae37d51754737747aaaf413b4d6b3b5fbacd52bdb2d0d6e5b2edd9aee SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020 CMAKE_VERSION=3.18.3 @@ -24,9 +25,9 @@ CMAKE_HASH=2c89f4e30af4914fd6fb5d00f863629812ada848eee4e2d29ec7e456d7fa32e5 CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download LIBXCRYPT_VERSION=4.4.17 -LIBXCRYPT_DOWNLOAD_URL=https://codeload.github.com/besser82/libxcrypt/tar.gz LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 +LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive -GIT_ROOT=git-2.29.1 -GIT_HASH=728845a66103d8d1572a656123c2c09d7aa4c0ab8f4c3dc3911e14e18c85ee67 +GIT_ROOT=git-2.30.0 +GIT_HASH=d24c4fa2a658318c2e66e25ab67cc30038a35696d2d39e6b12ceccf024de1e5e GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git diff --git a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_utils.sh b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_utils.sh index 5a36434e2c..6c780f7a50 100755 --- a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_utils.sh +++ b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/build_utils.sh @@ -175,9 +175,12 @@ function build_libtool { } function build_libxcrypt { - curl -fsSLO "$LIBXCRYPT_DOWNLOAD_URL"/v"$LIBXCRYPT_VERSION" - check_sha256sum "v$LIBXCRYPT_VERSION" "$LIBXCRYPT_HASH" - tar xfz "v$LIBXCRYPT_VERSION" + check_var ${LIBXCRYPT_VERSION} + check_var ${LIBXCRYPT_HASH} + check_var ${LIBXCRYPT_DOWNLOAD_URL} + fetch_source v${LIBXCRYPT_VERSION}.tar.gz ${LIBXCRYPT_DOWNLOAD_URL} + check_sha256sum "v${LIBXCRYPT_VERSION}.tar.gz" "$LIBXCRYPT_HASH" + tar xfz "v${LIBXCRYPT_VERSION}.tar.gz" pushd "libxcrypt-$LIBXCRYPT_VERSION" ./autogen.sh > /dev/null do_standard_install \ @@ -195,7 +198,7 @@ function build_libxcrypt { --disable-werror cp -P ./so.1/usr/local/lib/libcrypt.so.1* /usr/local/lib/ popd - rm -rf "v$LIBXCRYPT_VERSION" "libxcrypt-$LIBXCRYPT_VERSION" + rm -rf "v${LIBXCRYPT_VERSION}.tar.gz" "libxcrypt-$LIBXCRYPT_VERSION" # Delete GLIBC version headers and libraries rm -rf /usr/include/crypt.h @@ -205,10 +208,12 @@ function build_libxcrypt { function build_patchelf { local patchelf_version=$1 local patchelf_hash=$2 - local src_dir=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) - curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$patchelf_version.tar.gz - check_sha256sum patchelf.tar.gz $patchelf_hash - tar -xzf patchelf.tar.gz + check_var ${patchelf_version} + check_var ${patchelf_hash} + check_var ${PATCHELF_DOWNLOAD_URL} + fetch_source ${patchelf_version}.tar.gz ${PATCHELF_DOWNLOAD_URL} + check_sha256sum ${patchelf_version}.tar.gz $patchelf_hash + tar -xzf ${patchelf_version}.tar.gz (cd patchelf-$patchelf_version && ./bootstrap.sh && do_standard_install) - rm -rf patchelf.tar.gz patchelf-$patchelf_version + rm -rf ${patchelf_version}.tar.gz patchelf-$patchelf_version } diff --git a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements-tools.txt b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements-tools.txt index 44c6ded6ea..bbd2c99548 100644 --- a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements-tools.txt +++ b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements-tools.txt @@ -1,14 +1,12 @@ # pip requirements for tools # NOTE: certifi has GPG signatures; could download and verify independently. -certifi==2020.11.8 \ - --hash=sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd \ - --hash=sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4 -auditwheel==3.2.0 \ - --hash=sha256:374a42f5ed95aea02ce9e775bbc4abdf57e84054400bab063616f0ab9bb20ebe -# these packages required for auditwheel +certifi==2020.12.5 \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c +auditwheel==3.3.1 \ + --hash=sha256:489d9a8152aeb463d23922853b9a3fa1fd2a480daac4dcbd809e22c63c97344d \ + --hash=sha256:c57d1bc633881b16445874276e8a4becf535b170a30dd1d0a567b8d63d63db35 +# this package is required for auditwheel pyelftools==0.27 \ --hash=sha256:5609aa6da1123fccfae2e8431a67b4146aa7fad5b3889f808df12b110f230937 \ --hash=sha256:cde854e662774c5457d688ca41615f6594187ba7067af101232df889a6b7a66b -wheel==0.34.2 \ - --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e \ - --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 diff --git a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements.txt b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements.txt index 166fdfc999..31c6e82807 100644 --- a/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/manylinux2014_build_scripts/requirements.txt @@ -1,11 +1,14 @@ # pip requirements for all cpythons # NOTE: pip has GPG signatures; could download and verify independently. -pip==20.2.4 \ - --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ - --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -wheel==0.35.1 \ - --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ - --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f -setuptools==50.3.2 \ +pip==20.3.3 \ + --hash=sha256:fab098c8a1758295dd9f57413c199f23571e8fde6cc39c22c78c961b4ac6286d \ + --hash=sha256:79c1ac8a9dccbec8752761cb5a2df833224263ca661477a2a9ed03ddf4e0e3ba +wheel==0.36.2 \ + --hash=sha256:78b5b185f0e5763c26ca1e324373aadd49182ca90e825f7853f4b2509215dc0e \ + --hash=sha256:e11eefd162658ea59a60a0f6c7d493a7190ea4b9a85e335b33489d9f17e0245e +setuptools==50.3.2 ; python_version=='3.5' \ --hash=sha256:2c242a0856fbad7efbe560df4a7add9324f340cf48df43651e9604924466794a \ --hash=sha256:ed0519d27a243843b05d82a5e9d01b0b083d9934eaa3d02779a23da18077bd3c +setuptools==51.1.1 ; python_version>='3.6' \ + --hash=sha256:6d119767443a0f770bab9738b86ce9c0a699a7759ff4f61af583ee73d2e528a0 \ + --hash=sha256:0b43d1e0e0ac1467185581c2ceaf86b5c1a1bc408f8f6407687b0856302d1850