Delete extra cgmanifest entries and files (#23583)

Remove the auto-generated cgmanifest.json. Because now we can get the
same information from vcpkg.
Also, remote some outdated entries in the main cgmanifest.json file.
This commit is contained in:
Changming Sun 2025-02-05 11:21:21 -08:00 committed by GitHub
parent 5f6a3158f8
commit 8f6ddf3bd5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1 additions and 692 deletions

View file

@ -1,30 +1,3 @@
# CGManifest Files
This directory contains CGManifest (cgmanifest.json) files.
See [here](https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html) for details.
## `cgmanifests/generated/cgmanifest.json`
This file contains generated CGManifest entries.
It covers these dependencies:
- git submodules
- dependencies from the Dockerfile `tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11`
- the entries in [../cmake/deps.txt](../cmake/deps.txt)
If any of these dependencies change, this file should be updated.
**When updating, please regenerate instead of editing manually.**
### How to Generate
1. Change to the repository root directory.
2. Ensure the git submodules are checked out and up to date. For example, with:
```
$ git submodule update --init --recursive
```
3. Run the generator script:
```
$ python cgmanifests/generate_cgmanifest.py --username <xxx> --token <your_access_token>
```
Please supply your github username and access token to the script. If you don't have a token, you can generate one at https://github.com/settings/tokens. This is for authenticating with Github REST API so that you would not hit the rate limit.
## `cgmanifests/cgmanifest.json`
This file contains non-generated CGManifest entries. Please edit directly as needed.
See [here](https://docs.opensource.microsoft.com/tools/cg/cgmanifest.html) for details.

View file

@ -1,15 +1,6 @@
{
"$schema": "https://json.schemastore.org/component-detection-manifest.json",
"Registrations": [
{
"component": {
"type": "git",
"git": {
"commitHash": "215105818dfde3174fe799600bb0f3cae233d0bf",
"repositoryUrl": "https://github.com/abseil/abseil-cpp.git"
}
}
},
{
"component": {
"Type": "maven",
@ -131,16 +122,6 @@
}
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "b31f58de6fa8bbda5353b3c77d9be4914399724d",
"repositoryUrl": "https://github.com/pytorch/pytorch.git"
},
"comments": "pytorch 1.6 used by onnxruntime training image"
}
},
{
"component": {
"type": "git",
@ -189,15 +170,6 @@
}
}
},
{
"component": {
"git": {
"commitHash": "02a2a458ac15912d7d87cc1171e811b0c5219ece",
"repositoryUrl": "https://github.com/grpc/grpc"
},
"type": "git"
}
},
{
"component": {
"git": {
@ -324,116 +296,6 @@
"comments": "For Nodejs binding"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "aead4d751c2101e23336aa73f2380df83e7a13f3",
"repositoryUrl": "https://github.com/pypa/manylinux"
},
"comments": "For building our CI build docker image"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "c974557598645360fbabac71352b083117e3cc17",
"repositoryUrl": "https://gitlab.kitware.com/cmake/cmake"
},
"comments": "CMake 3.24.3. For building our CI build docker image"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "1e5d33e9b9b8631b36f061103a30208b206fd03a",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.9.1"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "6503f05dd59e26a9986bdea097b3da9b3546f45b",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.8.7"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "13c94747c74437e594b7fc242ff7da668e81887c",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.7.9"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "c0a9afe2ac1820409e6173bd1893ebee2cf50270",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.6.12"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "426b022776672fdf3d71ddd98d89af341c88080f",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.5.10"
}
},
{
"component": {
"type": "pip",
"pip": {
"Name": "transformers",
"Version": "4.38.0"
},
"comments": "Installed in the training docker image"
}
},
{
"component": {
"type": "pip",
"pip": {
"Name": "msgpack",
"Version": "1.0.0"
},
"comments": "Installed in the training docker image"
}
},
{
"component": {
"type": "pip",
"pip": {
"Name": "tensorboardX",
"Version": "1.8"
},
"comments": "Installed in the training docker image"
}
},
{
"component": {
"type": "pip",
"pip": {
"Name": "tensorboard",
"Version": "2.3.0"
},
"comments": "Installed in the training docker image"
}
},
{
"component": {
"type": "git",
@ -493,15 +355,6 @@
},
"comments": "python-pillow. Implementation logic for anti-aliasing copied by Resize CPU kernel."
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "e7248b26a1ed53fa030c5c459f7ea095dfd276ac",
"repositoryUrl": "https://gitlab.com/libeigen/eigen.git"
}
}
}
],
"Version": 1

View file

@ -1,159 +0,0 @@
#!/usr/bin/env python3
import argparse
import csv
import json
import os
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import PurePosixPath
from urllib.parse import urlparse
import requests
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument("--username", required=True, help="Github username")
parser.add_argument("--token", required=True, help="Github access token")
return parser.parse_args()
args = parse_arguments()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, ".."))
package_name = None
package_filename = None
package_url = None
registrations = []
@dataclass(frozen=True)
class GitDep:
commit: str
url: str
git_deps = {}
def add_github_dep(name, parsed_url):
segments = parsed_url.path.split("/")
org_name = segments[1]
repo_name = segments[2]
if segments[3] != "archive":
print("unrecognized github url path:" + parsed_url.path)
return
git_repo_url = f"https://github.com/{org_name}/{repo_name}.git"
# For example, the path might be like '/myorg/myrepo/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip'
# The last segment, segments[4], is '5a5f8a5935762397aa68429b5493084ff970f774.zip'
if len(segments) == 5 and re.match(r"[0-9a-f]{40}", PurePosixPath(segments[4]).stem):
commit = PurePosixPath(segments[4]).stem
dep = GitDep(commit, git_repo_url)
if dep not in git_deps:
git_deps[dep] = name
else:
# TODO: support urls like: https://github.com/onnx/onnx-tensorrt/archive/refs/tags/release/7.1.zip
if len(segments) == 5:
tag = PurePosixPath(segments[4]).stem
if tag.endswith(".tar"):
tag = PurePosixPath(tag).stem
elif segments[4] == "refs" and segments[5] == "tags":
tag = PurePosixPath(segments[6]).stem
if tag.endswith(".tar"):
tag = PurePosixPath(tag).stem
else:
print("unrecognized github url path:" + parsed_url.path)
return
# Make a REST call to convert to tag to a git commit
url = f"https://api.github.com/repos/{org_name}/{repo_name}/git/refs/tags/{tag}"
print(f"requesting {url} ...")
res = requests.get(url, auth=(args.username, args.token))
response_json = res.json()
tag_object = response_json["object"]
if tag_object["type"] == "commit":
commit = tag_object["sha"]
elif tag_object["type"] == "tag":
res = requests.get(tag_object["url"], auth=(args.username, args.token))
commit = res.json()["object"]["sha"]
else:
print("unrecognized github url path:" + parsed_url.path)
return
dep = GitDep(commit, git_repo_url)
if dep not in git_deps:
git_deps[dep] = name
def normalize_path_separators(path):
return path.replace(os.path.sep, "/")
proc = subprocess.run(
[
"git",
"submodule",
"foreach",
"--quiet",
"'{}' '{}' $toplevel/$sm_path".format(
normalize_path_separators(sys.executable),
normalize_path_separators(os.path.join(SCRIPT_DIR, "print_submodule_info.py")),
),
],
check=True,
cwd=REPO_DIR,
capture_output=True,
text=True,
)
submodule_lines = proc.stdout.splitlines()
for submodule_line in submodule_lines:
(absolute_path, url, commit) = submodule_line.split(" ")
git_deps[GitDep(commit, url)] = (
f"git submodule at {normalize_path_separators(os.path.relpath(absolute_path, REPO_DIR))}"
)
with open(os.path.join(SCRIPT_DIR, "..", "cmake", "deps.txt")) as f:
depfile_reader = csv.reader(f, delimiter=";")
for row in depfile_reader:
if len(row) < 3:
continue
name = row[0]
# Lines start with "#" are comments
if name.startswith("#"):
continue
url = row[1]
parsed_url = urlparse(url)
# TODO: add support for gitlab
if parsed_url.hostname == "github.com":
add_github_dep(name, parsed_url)
else:
print("unrecognized url:" + url)
for git_dep, comment in git_deps.items():
registration = {
"component": {
"type": "git",
"git": {
"commitHash": git_dep.commit,
"repositoryUrl": git_dep.url,
},
"comments": comment,
}
}
registrations.append(registration)
cgmanifest = {
"$schema": "https://json.schemastore.org/component-detection-manifest.json",
"Version": 1,
"Registrations": registrations,
}
with open(os.path.join(SCRIPT_DIR, "generated", "cgmanifest.json"), mode="w") as generated_cgmanifest_file:
print(json.dumps(cgmanifest, indent=2), file=generated_cgmanifest_file)

View file

@ -1,2 +0,0 @@
`cgmanifests/generated/cgmanifest.json` is a generated file. Please do not edit it directly.
More info [here](../README.md).

View file

@ -1,356 +0,0 @@
{
"$schema": "https://json.schemastore.org/component-detection-manifest.json",
"Version": 1,
"Registrations": [
{
"component": {
"type": "git",
"git": {
"commitHash": "d52c46520124845b1e0e0525f2759299d840143f",
"repositoryUrl": "https://github.com/emscripten-core/emsdk.git"
},
"comments": "git submodule at cmake/external/emsdk"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "7a2ed51a6b682a83e345ff49fc4cfd7ca47550db",
"repositoryUrl": "https://github.com/google/libprotobuf-mutator.git"
},
"comments": "git submodule at cmake/external/libprotobuf-mutator"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "b8baa8446686496da4cc8fda09f2b6fe65c2a02c",
"repositoryUrl": "https://github.com/onnx/onnx.git"
},
"comments": "git submodule at cmake/external/onnx"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "4447c7562e3bc702ade25105912dce503f0c4010",
"repositoryUrl": "https://github.com/abseil/abseil-cpp.git"
},
"comments": "abseil_cpp"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "dbb0094fd0cb936469e35320bf37e866ef7a1da4",
"repositoryUrl": "https://github.com/apple/coremltools.git"
},
"comments": "coremltools"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "3c73d91c0b04e2b59462f0a741be8c07024c1bc0",
"repositoryUrl": "https://github.com/jarro2783/cxxopts.git"
},
"comments": "cxxopts"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "6e921e1b1d21e84a5c82416ba7ecd98e33a436d0",
"repositoryUrl": "https://github.com/HowardHinnant/date.git"
},
"comments": "date"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "277508879878e0a5b5b43599b1bea11f66eb3c6c",
"repositoryUrl": "https://github.com/dmlc/dlpack.git"
},
"comments": "dlpack"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "0100f6a5779831fa7a651e4b67ef389a8752bd9b",
"repositoryUrl": "https://github.com/google/flatbuffers.git"
},
"comments": "flatbuffers"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "0a92994d729ff76a58f692d3028ca1b64b145d91",
"repositoryUrl": "https://github.com/Maratyszcza/FP16.git"
},
"comments": "fp16"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "63058eff77e11aa15bf531df5dd34395ec3017c8",
"repositoryUrl": "https://github.com/Maratyszcza/FXdiv.git"
},
"comments": "fxdiv"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "a6ad7fbbdc2e14fab82bb8a6d27760d700198cbf",
"repositoryUrl": "https://github.com/google/benchmark.git"
},
"comments": "google_benchmark"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "e39786088138f2749d64e9e90e0f9902daa77c40",
"repositoryUrl": "https://github.com/google/googletest.git"
},
"comments": "googletest"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "fe98e0b93565382648129271381c14d6205255e3",
"repositoryUrl": "https://github.com/google/XNNPACK.git"
},
"comments": "googlexnnpack"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "4f8fba14066156b73f1189a2b8bd568bde5284c5",
"repositoryUrl": "https://github.com/nlohmann/json.git"
},
"comments": "json"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "a3534567187d2edc428efd3f13466ff75fe5805c",
"repositoryUrl": "https://github.com/microsoft/GSL.git"
},
"comments": "microsoft_gsl"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "55f373e164d3f092dd6c7a56e3de6f90c4c6f3dc",
"repositoryUrl": "https://github.com/microsoft/wil.git"
},
"comments": "microsoft_wil"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "3e313478d91c04ac5821743688ce55fc27432c4f",
"repositoryUrl": "https://github.com/microsoft/mimalloc.git"
},
"comments": "mimalloc"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "0a0b5fb001ce0233ae3a6f99d849c0649e5a7361",
"repositoryUrl": "https://github.com/boostorg/mp11.git"
},
"comments": "mp11"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "9c69a24bc2e20c8a511a4e6b06fd49639ec5300a",
"repositoryUrl": "https://github.com/onnx/onnx-tensorrt.git"
},
"comments": "onnx_tensorrt"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c",
"repositoryUrl": "https://github.com/protocolbuffers/protobuf.git"
},
"comments": "protobuf"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "072586a71b55b7f8c584153d223e95687148a900",
"repositoryUrl": "https://github.com/Maratyszcza/psimd.git"
},
"comments": "psimd"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "4e80ca24521aa0fb3a746f9ea9c3eaa20e9afbb0",
"repositoryUrl": "https://github.com/google/pthreadpool.git"
},
"comments": "pthreadpool"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "941f45bcb51457884fa1afd6e24a67377d70f75c",
"repositoryUrl": "https://github.com/pybind/pybind11.git"
},
"comments": "pybind11"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "8a1772a0c5c447df2d18edf33ec4603a8c9c04a6",
"repositoryUrl": "https://github.com/pytorch/cpuinfo.git"
},
"comments": "pytorch_cpuinfo"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "6dcd83d60f7944926bfd308cc13979fc53dd69ca",
"repositoryUrl": "https://github.com/google/re2.git"
},
"comments": "re2"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "4cafc9196c4da9c817992b20f5253ef967685bf8",
"repositoryUrl": "https://github.com/dcleblanc/SafeInt.git"
},
"comments": "safeint"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81",
"repositoryUrl": "https://github.com/tensorflow/tensorboard.git"
},
"comments": "tensorboard"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "f7b19de32c5d1f3cedfc735c2849f12b537522ee",
"repositoryUrl": "https://github.com/NVIDIA/cutlass.git"
},
"comments": "cutlass"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "72c943dea2b9240cd09efde15191e144bc7c7d38",
"repositoryUrl": "https://github.com/protocolbuffers/utf8_range.git"
},
"comments": "utf8_range"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "f3f6caa6e8adb420e005ec41c6fefc8d75affb6e",
"repositoryUrl": "https://github.com/microsoft/onnxruntime-extensions.git"
},
"comments": "extensions"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "204da9c522cebec5220bba52cd3542ebcaf99e7a",
"repositoryUrl": "https://github.com/ROCmSoftwarePlatform/composable_kernel.git"
},
"comments": "composable_kernel"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "de28d93dfa9ebf3e473127c1c657e1920a5345ee",
"repositoryUrl": "https://github.com/microsoft/DirectX-Headers.git"
},
"comments": "directx_headers"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "de355c7094af70467f2b264f531ab5c5f4401c42",
"repositoryUrl": "https://github.com/NVIDIA/cudnn-frontend.git"
},
"comments": "cudnn_frontend"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "b9b4a37041dec3dd62ac92014a6cc1aece48d9f3",
"repositoryUrl": "https://github.com/google/dawn.git"
},
"comments": "dawn"
}
}
]
}