saymrwulf/transformers
1
0
Fork 0
mirror of https://github.com/saymrwulf/transformers.git synced 2026-05-14 20:58:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

0001

Browse source
This commit is contained in:
ydshieh 2025-01-08 15:47:21 +01:00
parent 4d1d489617
commit a5ae82d1cf
3 changed files with 20 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.circleci/config.yml
View file

@ -191,9 +191,6 @@ workflows:
- equal: [<<pipeline.project.git_url>>, https://github.com/huggingface/transformers]
- not: <<pipeline.parameters.nightly>>
jobs:
- check_circleci_user
- check_code_quality
- check_repository_consistency
- fetch_tests
setup_and_quality_2:

7
.circleci/create_circleci_config.py
View file

@ -154,7 +154,7 @@ class CircleCIJob:
},
{"run": {
"name": "Run tests",
"command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {additional_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"}
"command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n 1 {additional_flags} {' '.join(pytest_flags)} tests/models/flaubert/test_modeling_flaubert.py::FlaubertModelTest::test_batching_equivalence | tee tests_output.txt)"}
},
{"run": {"name": "Expand to show skipped tests", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --skip"}},
{"run": {"name": "Failed tests: show reasons", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --fail"}},
@ -198,7 +198,7 @@ torch_job = CircleCIJob(
"torch",
docker_image=[{"image": "huggingface/transformers-torch-light"}],
marker="not generate",
parallelism=6,
parallelism=1,
)
generate_job = CircleCIJob(
@ -358,7 +358,8 @@ EXAMPLES_TESTS = [examples_torch_job, examples_tensorflow_job]
PIPELINE_TESTS = [pipelines_torch_job, pipelines_tf_job]
REPO_UTIL_TESTS = [repo_utils_job]
DOC_TESTS = [doc_test_job]
ALL_TESTS = REGULAR_TESTS + EXAMPLES_TESTS + PIPELINE_TESTS + REPO_UTIL_TESTS + DOC_TESTS + [custom_tokenizers_job] + [exotic_models_job] # fmt: skip
# ALL_TESTS = REGULAR_TESTS + EXAMPLES_TESTS + PIPELINE_TESTS + REPO_UTIL_TESTS + DOC_TESTS + [custom_tokenizers_job] + [exotic_models_job] # fmt: skip
ALL_TESTS = [torch_job]
def create_circleci_config(folder=None):

32
utils/tests_fetcher.py
View file

@ -1134,23 +1134,23 @@ def parse_commit_message(commit_message: str) -> Dict[str, bool]:
JOB_TO_TEST_FILE = {
"tests_torch_and_tf": r"tests/models/.*/test_modeling_(?:tf_|(?!flax)).*",
"tests_torch_and_flax": r"tests/models/.*/test_modeling_(?:flax|(?!tf)).*",
"tests_tf": r"tests/models/.*/test_modeling_tf_.*",
# "tests_torch_and_tf": r"tests/models/.*/test_modeling_(?:tf_|(?!flax)).*",
# "tests_torch_and_flax": r"tests/models/.*/test_modeling_(?:flax|(?!tf)).*",
# "tests_tf": r"tests/models/.*/test_modeling_tf_.*",
"tests_torch": r"tests/models/.*/test_modeling_(?!(?:flax_|tf_)).*",
"tests_generate": r"tests/models/.*/test_modeling_(?!(?:flax_|tf_)).*",
"tests_tokenization": r"tests/models/.*/test_tokenization.*",
"tests_processors": r"tests/models/.*/test_(?!(?:modeling_|tokenization_)).*", # takes feature extractors, image processors, processors
"examples_torch": r"examples/pytorch/.*test_.*",
"examples_tensorflow": r"examples/tensorflow/.*test_.*",
"tests_exotic_models": r"tests/models/.*(?=layoutlmv|nat|deta|udop|nougat).*",
"tests_custom_tokenizers": r"tests/models/.*/test_tokenization_(?=bert_japanese|openai|clip).*",
# "repo_utils": r"tests/[^models].*test.*", TODO later on we might want to do
"pipelines_tf": r"tests/models/.*/test_modeling_tf_.*",
"pipelines_torch": r"tests/models/.*/test_modeling_(?!(?:flax_|tf_)).*",
"tests_hub": r"tests/.*",
"tests_onnx": r"tests/models/.*/test_modeling_(?:tf_|(?!flax)).*",
"tests_non_model": r"tests/[^/]*?/test_.*\.py",
# "tests_generate": r"tests/models/.*/test_modeling_(?!(?:flax_|tf_)).*",
# "tests_tokenization": r"tests/models/.*/test_tokenization.*",
# "tests_processors": r"tests/models/.*/test_(?!(?:modeling_|tokenization_)).*", # takes feature extractors, image processors, processors
# "examples_torch": r"examples/pytorch/.*test_.*",
# "examples_tensorflow": r"examples/tensorflow/.*test_.*",
# "tests_exotic_models": r"tests/models/.*(?=layoutlmv|nat|deta|udop|nougat).*",
# "tests_custom_tokenizers": r"tests/models/.*/test_tokenization_(?=bert_japanese|openai|clip).*",
# # "repo_utils": r"tests/[^models].*test.*", TODO later on we might want to do
# "pipelines_tf": r"tests/models/.*/test_modeling_tf_.*",
# "pipelines_torch": r"tests/models/.*/test_modeling_(?!(?:flax_|tf_)).*",
# "tests_hub": r"tests/.*",
# "tests_onnx": r"tests/models/.*/test_modeling_(?:tf_|(?!flax)).*",
# "tests_non_model": r"tests/[^/]*?/test_.*\.py",
}