diff --git a/cgmanifests/README.md b/cgmanifests/README.md index fcd15232f9..a7d816a401 100644 --- a/cgmanifests/README.md +++ b/cgmanifests/README.md @@ -1,30 +1,3 @@ # CGManifest Files This directory contains CGManifest (cgmanifest.json) files. -See [here](https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html) for details. - -## `cgmanifests/generated/cgmanifest.json` -This file contains generated CGManifest entries. - -It covers these dependencies: -- git submodules -- dependencies from the Dockerfile `tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11` -- the entries in [../cmake/deps.txt](../cmake/deps.txt) - -If any of these dependencies change, this file should be updated. -**When updating, please regenerate instead of editing manually.** - -### How to Generate -1. Change to the repository root directory. -2. Ensure the git submodules are checked out and up to date. For example, with: - ``` - $ git submodule update --init --recursive - ``` -3. Run the generator script: - ``` - $ python cgmanifests/generate_cgmanifest.py --username --token - ``` - -Please supply your github username and access token to the script. If you don't have a token, you can generate one at https://github.com/settings/tokens. This is for authenticating with Github REST API so that you would not hit the rate limit. - -## `cgmanifests/cgmanifest.json` -This file contains non-generated CGManifest entries. Please edit directly as needed. +See [here](https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html) for details. \ No newline at end of file diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index 46349f4392..d883f89dfd 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -1,15 +1,6 @@ { "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ - { - "component": { - "type": "git", - "git": { - "commitHash": "215105818dfde3174fe799600bb0f3cae233d0bf", - "repositoryUrl": "https://github.com/abseil/abseil-cpp.git" - } - } - }, { "component": { "Type": "maven", @@ -131,16 +122,6 @@ } } }, - { - "component": { - "type": "git", - "git": { - "commitHash": "b31f58de6fa8bbda5353b3c77d9be4914399724d", - "repositoryUrl": "https://github.com/pytorch/pytorch.git" - }, - "comments": "pytorch 1.6 used by onnxruntime training image" - } - }, { "component": { "type": "git", @@ -189,15 +170,6 @@ } } }, - { - "component": { - "git": { - "commitHash": "02a2a458ac15912d7d87cc1171e811b0c5219ece", - "repositoryUrl": "https://github.com/grpc/grpc" - }, - "type": "git" - } - }, { "component": { "git": { @@ -324,116 +296,6 @@ "comments": "For Nodejs binding" } }, - { - "component": { - "type": "git", - "git": { - "commitHash": "aead4d751c2101e23336aa73f2380df83e7a13f3", - "repositoryUrl": "https://github.com/pypa/manylinux" - }, - "comments": "For building our CI build docker image" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "c974557598645360fbabac71352b083117e3cc17", - "repositoryUrl": "https://gitlab.kitware.com/cmake/cmake" - }, - "comments": "CMake 3.24.3. For building our CI build docker image" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "1e5d33e9b9b8631b36f061103a30208b206fd03a", - "repositoryUrl": "https://github.com/python/cpython" - }, - "comments": "Python 3.9.1" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "6503f05dd59e26a9986bdea097b3da9b3546f45b", - "repositoryUrl": "https://github.com/python/cpython" - }, - "comments": "Python 3.8.7" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "13c94747c74437e594b7fc242ff7da668e81887c", - "repositoryUrl": "https://github.com/python/cpython" - }, - "comments": "Python 3.7.9" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "c0a9afe2ac1820409e6173bd1893ebee2cf50270", - "repositoryUrl": "https://github.com/python/cpython" - }, - "comments": "Python 3.6.12" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "426b022776672fdf3d71ddd98d89af341c88080f", - "repositoryUrl": "https://github.com/python/cpython" - }, - "comments": "Python 3.5.10" - } - }, - { - "component": { - "type": "pip", - "pip": { - "Name": "transformers", - "Version": "4.38.0" - }, - "comments": "Installed in the training docker image" - } - }, - { - "component": { - "type": "pip", - "pip": { - "Name": "msgpack", - "Version": "1.0.0" - }, - "comments": "Installed in the training docker image" - } - }, - { - "component": { - "type": "pip", - "pip": { - "Name": "tensorboardX", - "Version": "1.8" - }, - "comments": "Installed in the training docker image" - } - }, - { - "component": { - "type": "pip", - "pip": { - "Name": "tensorboard", - "Version": "2.3.0" - }, - "comments": "Installed in the training docker image" - } - }, { "component": { "type": "git", @@ -493,15 +355,6 @@ }, "comments": "python-pillow. Implementation logic for anti-aliasing copied by Resize CPU kernel." } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "e7248b26a1ed53fa030c5c459f7ea095dfd276ac", - "repositoryUrl": "https://gitlab.com/libeigen/eigen.git" - } - } } ], "Version": 1 diff --git a/cgmanifests/generate_cgmanifest.py b/cgmanifests/generate_cgmanifest.py deleted file mode 100644 index b2e8f6816a..0000000000 --- a/cgmanifests/generate_cgmanifest.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import csv -import json -import os -import re -import subprocess -import sys -from dataclasses import dataclass -from pathlib import PurePosixPath -from urllib.parse import urlparse - -import requests - - -def parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("--username", required=True, help="Github username") - parser.add_argument("--token", required=True, help="Github access token") - - return parser.parse_args() - - -args = parse_arguments() - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..")) - -package_name = None -package_filename = None -package_url = None - -registrations = [] - - -@dataclass(frozen=True) -class GitDep: - commit: str - url: str - - -git_deps = {} - - -def add_github_dep(name, parsed_url): - segments = parsed_url.path.split("/") - org_name = segments[1] - repo_name = segments[2] - if segments[3] != "archive": - print("unrecognized github url path:" + parsed_url.path) - return - git_repo_url = f"https://github.com/{org_name}/{repo_name}.git" - # For example, the path might be like '/myorg/myrepo/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip' - # The last segment, segments[4], is '5a5f8a5935762397aa68429b5493084ff970f774.zip' - if len(segments) == 5 and re.match(r"[0-9a-f]{40}", PurePosixPath(segments[4]).stem): - commit = PurePosixPath(segments[4]).stem - dep = GitDep(commit, git_repo_url) - if dep not in git_deps: - git_deps[dep] = name - else: - # TODO: support urls like: https://github.com/onnx/onnx-tensorrt/archive/refs/tags/release/7.1.zip - if len(segments) == 5: - tag = PurePosixPath(segments[4]).stem - if tag.endswith(".tar"): - tag = PurePosixPath(tag).stem - elif segments[4] == "refs" and segments[5] == "tags": - tag = PurePosixPath(segments[6]).stem - if tag.endswith(".tar"): - tag = PurePosixPath(tag).stem - else: - print("unrecognized github url path:" + parsed_url.path) - return - # Make a REST call to convert to tag to a git commit - url = f"https://api.github.com/repos/{org_name}/{repo_name}/git/refs/tags/{tag}" - print(f"requesting {url} ...") - res = requests.get(url, auth=(args.username, args.token)) - response_json = res.json() - tag_object = response_json["object"] - if tag_object["type"] == "commit": - commit = tag_object["sha"] - elif tag_object["type"] == "tag": - res = requests.get(tag_object["url"], auth=(args.username, args.token)) - commit = res.json()["object"]["sha"] - else: - print("unrecognized github url path:" + parsed_url.path) - return - dep = GitDep(commit, git_repo_url) - if dep not in git_deps: - git_deps[dep] = name - - -def normalize_path_separators(path): - return path.replace(os.path.sep, "/") - - -proc = subprocess.run( - [ - "git", - "submodule", - "foreach", - "--quiet", - "'{}' '{}' $toplevel/$sm_path".format( - normalize_path_separators(sys.executable), - normalize_path_separators(os.path.join(SCRIPT_DIR, "print_submodule_info.py")), - ), - ], - check=True, - cwd=REPO_DIR, - capture_output=True, - text=True, -) - - -submodule_lines = proc.stdout.splitlines() -for submodule_line in submodule_lines: - (absolute_path, url, commit) = submodule_line.split(" ") - git_deps[GitDep(commit, url)] = ( - f"git submodule at {normalize_path_separators(os.path.relpath(absolute_path, REPO_DIR))}" - ) - -with open(os.path.join(SCRIPT_DIR, "..", "cmake", "deps.txt")) as f: - depfile_reader = csv.reader(f, delimiter=";") - for row in depfile_reader: - if len(row) < 3: - continue - name = row[0] - # Lines start with "#" are comments - if name.startswith("#"): - continue - url = row[1] - parsed_url = urlparse(url) - # TODO: add support for gitlab - if parsed_url.hostname == "github.com": - add_github_dep(name, parsed_url) - else: - print("unrecognized url:" + url) - -for git_dep, comment in git_deps.items(): - registration = { - "component": { - "type": "git", - "git": { - "commitHash": git_dep.commit, - "repositoryUrl": git_dep.url, - }, - "comments": comment, - } - } - registrations.append(registration) - -cgmanifest = { - "$schema": "https://json.schemastore.org/component-detection-manifest.json", - "Version": 1, - "Registrations": registrations, -} - -with open(os.path.join(SCRIPT_DIR, "generated", "cgmanifest.json"), mode="w") as generated_cgmanifest_file: - print(json.dumps(cgmanifest, indent=2), file=generated_cgmanifest_file) diff --git a/cgmanifests/generated/README.md b/cgmanifests/generated/README.md deleted file mode 100644 index 299ce4bc82..0000000000 --- a/cgmanifests/generated/README.md +++ /dev/null @@ -1,2 +0,0 @@ -`cgmanifests/generated/cgmanifest.json` is a generated file. Please do not edit it directly. -More info [here](../README.md). diff --git a/cgmanifests/generated/cgmanifest.json b/cgmanifests/generated/cgmanifest.json deleted file mode 100644 index 46f8c8891d..0000000000 --- a/cgmanifests/generated/cgmanifest.json +++ /dev/null @@ -1,356 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/component-detection-manifest.json", - "Version": 1, - "Registrations": [ - { - "component": { - "type": "git", - "git": { - "commitHash": "d52c46520124845b1e0e0525f2759299d840143f", - "repositoryUrl": "https://github.com/emscripten-core/emsdk.git" - }, - "comments": "git submodule at cmake/external/emsdk" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "7a2ed51a6b682a83e345ff49fc4cfd7ca47550db", - "repositoryUrl": "https://github.com/google/libprotobuf-mutator.git" - }, - "comments": "git submodule at cmake/external/libprotobuf-mutator" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "b8baa8446686496da4cc8fda09f2b6fe65c2a02c", - "repositoryUrl": "https://github.com/onnx/onnx.git" - }, - "comments": "git submodule at cmake/external/onnx" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "4447c7562e3bc702ade25105912dce503f0c4010", - "repositoryUrl": "https://github.com/abseil/abseil-cpp.git" - }, - "comments": "abseil_cpp" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "dbb0094fd0cb936469e35320bf37e866ef7a1da4", - "repositoryUrl": "https://github.com/apple/coremltools.git" - }, - "comments": "coremltools" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "3c73d91c0b04e2b59462f0a741be8c07024c1bc0", - "repositoryUrl": "https://github.com/jarro2783/cxxopts.git" - }, - "comments": "cxxopts" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "6e921e1b1d21e84a5c82416ba7ecd98e33a436d0", - "repositoryUrl": "https://github.com/HowardHinnant/date.git" - }, - "comments": "date" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "277508879878e0a5b5b43599b1bea11f66eb3c6c", - "repositoryUrl": "https://github.com/dmlc/dlpack.git" - }, - "comments": "dlpack" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "0100f6a5779831fa7a651e4b67ef389a8752bd9b", - "repositoryUrl": "https://github.com/google/flatbuffers.git" - }, - "comments": "flatbuffers" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "0a92994d729ff76a58f692d3028ca1b64b145d91", - "repositoryUrl": "https://github.com/Maratyszcza/FP16.git" - }, - "comments": "fp16" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "63058eff77e11aa15bf531df5dd34395ec3017c8", - "repositoryUrl": "https://github.com/Maratyszcza/FXdiv.git" - }, - "comments": "fxdiv" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "a6ad7fbbdc2e14fab82bb8a6d27760d700198cbf", - "repositoryUrl": "https://github.com/google/benchmark.git" - }, - "comments": "google_benchmark" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "e39786088138f2749d64e9e90e0f9902daa77c40", - "repositoryUrl": "https://github.com/google/googletest.git" - }, - "comments": "googletest" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "fe98e0b93565382648129271381c14d6205255e3", - "repositoryUrl": "https://github.com/google/XNNPACK.git" - }, - "comments": "googlexnnpack" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "4f8fba14066156b73f1189a2b8bd568bde5284c5", - "repositoryUrl": "https://github.com/nlohmann/json.git" - }, - "comments": "json" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "a3534567187d2edc428efd3f13466ff75fe5805c", - "repositoryUrl": "https://github.com/microsoft/GSL.git" - }, - "comments": "microsoft_gsl" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "55f373e164d3f092dd6c7a56e3de6f90c4c6f3dc", - "repositoryUrl": "https://github.com/microsoft/wil.git" - }, - "comments": "microsoft_wil" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "3e313478d91c04ac5821743688ce55fc27432c4f", - "repositoryUrl": "https://github.com/microsoft/mimalloc.git" - }, - "comments": "mimalloc" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "0a0b5fb001ce0233ae3a6f99d849c0649e5a7361", - "repositoryUrl": "https://github.com/boostorg/mp11.git" - }, - "comments": "mp11" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "9c69a24bc2e20c8a511a4e6b06fd49639ec5300a", - "repositoryUrl": "https://github.com/onnx/onnx-tensorrt.git" - }, - "comments": "onnx_tensorrt" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c", - "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git" - }, - "comments": "protobuf" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "072586a71b55b7f8c584153d223e95687148a900", - "repositoryUrl": "https://github.com/Maratyszcza/psimd.git" - }, - "comments": "psimd" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "4e80ca24521aa0fb3a746f9ea9c3eaa20e9afbb0", - "repositoryUrl": "https://github.com/google/pthreadpool.git" - }, - "comments": "pthreadpool" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "941f45bcb51457884fa1afd6e24a67377d70f75c", - "repositoryUrl": "https://github.com/pybind/pybind11.git" - }, - "comments": "pybind11" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "8a1772a0c5c447df2d18edf33ec4603a8c9c04a6", - "repositoryUrl": "https://github.com/pytorch/cpuinfo.git" - }, - "comments": "pytorch_cpuinfo" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "6dcd83d60f7944926bfd308cc13979fc53dd69ca", - "repositoryUrl": "https://github.com/google/re2.git" - }, - "comments": "re2" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "4cafc9196c4da9c817992b20f5253ef967685bf8", - "repositoryUrl": "https://github.com/dcleblanc/SafeInt.git" - }, - "comments": "safeint" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81", - "repositoryUrl": "https://github.com/tensorflow/tensorboard.git" - }, - "comments": "tensorboard" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "f7b19de32c5d1f3cedfc735c2849f12b537522ee", - "repositoryUrl": "https://github.com/NVIDIA/cutlass.git" - }, - "comments": "cutlass" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "72c943dea2b9240cd09efde15191e144bc7c7d38", - "repositoryUrl": "https://github.com/protocolbuffers/utf8_range.git" - }, - "comments": "utf8_range" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "f3f6caa6e8adb420e005ec41c6fefc8d75affb6e", - "repositoryUrl": "https://github.com/microsoft/onnxruntime-extensions.git" - }, - "comments": "extensions" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "204da9c522cebec5220bba52cd3542ebcaf99e7a", - "repositoryUrl": "https://github.com/ROCmSoftwarePlatform/composable_kernel.git" - }, - "comments": "composable_kernel" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "de28d93dfa9ebf3e473127c1c657e1920a5345ee", - "repositoryUrl": "https://github.com/microsoft/DirectX-Headers.git" - }, - "comments": "directx_headers" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "de355c7094af70467f2b264f531ab5c5f4401c42", - "repositoryUrl": "https://github.com/NVIDIA/cudnn-frontend.git" - }, - "comments": "cudnn_frontend" - } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "b9b4a37041dec3dd62ac92014a6cc1aece48d9f3", - "repositoryUrl": "https://github.com/google/dawn.git" - }, - "comments": "dawn" - } - } - ] -}