From fa8a9015bd74a166f1517cd61f933353349a4f8c Mon Sep 17 00:00:00 2001 From: baijumeswani Date: Thu, 25 Feb 2021 13:30:14 -0800 Subject: [PATCH] Mount hf model cache and use cache for loading hf models (#6810) --- .../python/orttraining_ortmodule_tests.py | 25 +++++++++++++------ ...g-linux-gpu-ortmodule-test-ci-pipeline.yml | 7 +++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py b/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py index 1de1b10bba..d3ec796da6 100644 --- a/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py +++ b/orttraining/orttraining/test/python/orttraining_ortmodule_tests.py @@ -20,15 +20,22 @@ def parse_arguments(): parser.add_argument("--cwd", help="Path to the current working directory") parser.add_argument("--mnist", help="Path to the mnist data directory", type=str, default=None) parser.add_argument("--bert_data", help="Path to the bert data directory", type=str, default=None) + parser.add_argument("--transformers_cache", help="Path to the transformers model cache directory", type=str, default=None) return parser.parse_args() -def run_ortmodule_api_tests(cwd, log): +def get_env_with_transformers_cache(transformers_cache): + return {'TRANSFORMERS_CACHE': transformers_cache} if transformers_cache else {} + + +def run_ortmodule_api_tests(cwd, log, transformers_cache): log.debug('Running: ORTModule-API tests') + env = get_env_with_transformers_cache(transformers_cache) + command = [sys.executable, '-m', 'pytest', '-sv', 'orttraining_test_ortmodule_api.py'] - run_subprocess(command, cwd=cwd, log=log).check_returncode() + run_subprocess(command, cwd=cwd, log=log, env=env).check_returncode() def run_ortmodule_poc_net(cwd, log, no_cuda, data_dir): @@ -56,9 +63,11 @@ def run_ortmodule_torch_lightning(cwd, log, data_dir): run_subprocess(command, cwd=cwd, log=log).check_returncode() -def run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda, data_dir): +def run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda, data_dir, transformers_cache): log.debug('Running: ORTModule HuggingFace BERT for sequence classification with --no-cuda arg {}.'.format(no_cuda)) + env = get_env_with_transformers_cache(transformers_cache) + command = [sys.executable, 'orttraining_test_ortmodule_bert_classifier.py'] if no_cuda: command.extend(['--no-cuda', '--epochs', str(3)]) @@ -66,7 +75,7 @@ def run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, if data_dir: command.extend(['--data_dir', data_dir]) - run_subprocess(command, cwd=cwd, log=log).check_returncode() + run_subprocess(command, cwd=cwd, log=log, env=env).check_returncode() def main(): @@ -75,15 +84,17 @@ def main(): log.info("Running ortmodule tests pipeline") - run_ortmodule_api_tests(cwd, log) + run_ortmodule_api_tests(cwd, log, transformers_cache=args.transformers_cache) run_ortmodule_poc_net(cwd, log, no_cuda=False, data_dir=args.mnist) run_ortmodule_poc_net(cwd, log, no_cuda=True, data_dir=args.mnist) - run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda=False, data_dir=args.bert_data) + run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda=False, + data_dir=args.bert_data, transformers_cache=args.transformers_cache) - run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda=True, data_dir=args.bert_data) + run_ort_module_hf_bert_for_sequence_classification_from_pretrained(cwd, log, no_cuda=True, + data_dir=args.bert_data, transformers_cache=args.transformers_cache) # TODO: Re-enable when PyTorch Lightning works with newer torchtext (nightlies after 2021-02-19) # run_ortmodule_torch_lightning(cwd, log, args.args.mnist) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml index 1ee2de1a96..5dc7c3923f 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml @@ -33,6 +33,10 @@ jobs: displayName: 'Mount bert-data' condition: succeededOrFailed() + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" + displayName: 'Mount hf-models-cache' + condition: succeededOrFailed() + # Entry point for all ORTModule tests - script: | docker run \ @@ -43,9 +47,10 @@ jobs: --volume $(Build.BinariesDirectory):/build \ --volume /mnist:/mnist \ --volume /bert_data:/bert_data \ + --volume /hf_models_cache:/hf_models_cache \ onnxruntime_ortmodule_tests_image \ /build/RelWithDebInfo/launch_test.py \ - --cmd_line_with_args "python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw" \ + --cmd_line_with_args "python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw --transformers_cache /hf_models_cache/huggingface/transformers" \ --cwd /build/RelWithDebInfo \ displayName: 'Run orttraining_ortmodule_tests.py' condition: succeededOrFailed()