From 92662659ba2da60f224aa6d608afba8044a39348 Mon Sep 17 00:00:00 2001 From: liqunfu Date: Tue, 27 Oct 2020 21:27:37 -0700 Subject: [PATCH] Liqun/remove number matching (#5606) replace number matching with relaxed comparison in frontend tests Co-authored-by: liqun --- .../test/python/orttraining_run_glue.py | 140 +++++++----------- .../python/orttraining_run_multiple_choice.py | 50 ++----- .../python/orttraining_transformer_trainer.py | 110 +++++--------- .../tools/ci_test/download_e2e_test_data.py | 15 +- tools/ci_build/build.py | 7 - ...raining-linux-gpu-e2e-test-ci-pipeline.yml | 6 +- ...ng-linux-gpu-frontend-test-ci-pipeline.yml | 2 - .../docker/scripts/training/requirements.txt | 8 +- 8 files changed, 117 insertions(+), 221 deletions(-) diff --git a/orttraining/orttraining/test/python/orttraining_run_glue.py b/orttraining/orttraining/test/python/orttraining_run_glue.py index d5954ecc3c..cf7c3fcbfd 100644 --- a/orttraining/orttraining/test/python/orttraining_run_glue.py +++ b/orttraining/orttraining/test/python/orttraining_run_glue.py @@ -25,7 +25,15 @@ from transformers import ( import onnxruntime from onnxruntime.capi.ort_trainer import ORTTrainer, LossScaler, ModelDescription, IODescription -from onnxruntime.capi._pybind_state import get_mpi_context_local_rank, get_mpi_context_local_size, get_mpi_context_world_rank, get_mpi_context_world_size + +try: + from onnxruntime.capi._pybind_state import get_mpi_context_local_rank, get_mpi_context_local_size,\ + get_mpi_context_world_rank, get_mpi_context_world_size + has_get_mpi_context_internal_api = True +except ImportError: + has_get_mpi_context_internal_api = False + pass + from orttraining_transformer_trainer import ORTTransformerTrainer @@ -74,93 +82,59 @@ class ORTGlueTest(unittest.TestCase): self.output_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "glue_test_output/") self.cache_dir = '/tmp/glue/' self.logging_steps = 10 - self.rtol = 1e-02 def test_roberta_with_mrpc(self): - expected_acc = 0.8676470588235294 - expected_f1 = 0.9035714285714286 - expected_acc_and_f1 = 0.885609243697479 - expected_loss = 0.3022572344862947 + expected_acc = 0.85 + expected_f1 = 0.88 + expected_loss = 0.35 + results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=False) - results_per_api = dict() - for use_new_api in [True, False]: - results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=False, use_new_api=use_new_api) - assert_allclose(results['acc'], expected_acc, rtol=self.rtol) - assert_allclose(results['f1'], expected_f1, rtol=self.rtol) - assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol) - assert_allclose(results['loss'], expected_loss, rtol=self.rtol) - results_per_api[use_new_api] = results - - verify_old_and_new_api_are_equal(results_per_api) + assert(results['acc'] >= expected_acc) + assert(results['f1'] >= expected_f1) + assert(results['loss'] <= expected_loss) def test_roberta_fp16_with_mrpc(self): - expected_acc = 0.8897058823529411 - expected_f1 = 0.9197860962566845 - expected_acc_and_f1 = 0.9047459893048129 - expected_loss = 0.3035417107098243 + expected_acc = 0.87 + expected_f1 = 0.90 + expected_loss = 0.33 - results_per_api = dict() - for use_new_api in [True, False]: - results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=True, use_new_api=use_new_api) - assert_allclose(results['acc'], expected_acc, rtol=self.rtol) - assert_allclose(results['f1'], expected_f1, rtol=self.rtol) - assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol) - assert_allclose(results['loss'], expected_loss, rtol=self.rtol) - results_per_api[use_new_api] = results + results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=True) - verify_old_and_new_api_are_equal(results_per_api) + assert(results['acc'] >= expected_acc) + assert(results['f1'] >= expected_f1) + assert(results['loss'] <= expected_loss) def test_bert_with_mrpc(self): if self.local_rank == -1: - expected_acc = 0.8553921568627451 - expected_f1 = 0.8970331588132635 - expected_acc_and_f1 = 0.8762126578380043 - expected_loss = 0.42737212419217707 + expected_acc = 0.83 + expected_f1 = 0.88 + expected_loss = 0.44 elif self.local_rank == 0: - expected_acc = 0.8308823529411765 - expected_f1 = 0.881646655231561 - expected_acc_and_f1 = 0.8562645040863688 - expected_loss = 0.42491564023144107 + expected_acc = 0.81 + expected_f1 = 0.86 + expected_loss = 0.44 - if self.local_rank == -1: - # not parallel case, we can run both new and old api tests - results_per_api = dict() - for use_new_api in [True, False]: - results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=False, use_new_api=use_new_api) - results_per_api[use_new_api] = results - - verify_old_and_new_api_are_equal(results_per_api) - else: - # with parallel training, TrainingArguments can only be created once (due to its cached _setup_devices) - # thus we can only choose one test case to run. - results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=False, use_new_api=True) + results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=False) if self.local_rank in [-1, 0]: - assert_allclose(results['acc'], expected_acc, rtol=self.rtol) - assert_allclose(results['f1'], expected_f1, rtol=self.rtol) - assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol) - assert_allclose(results['loss'], expected_loss, rtol=self.rtol) + assert(results['acc'] >= expected_acc) + assert(results['f1'] >= expected_f1) + assert(results['loss'] <= expected_loss) def test_bert_fp16_with_mrpc(self): - expected_acc = 0.8529411764705882 - expected_f1 = 0.8972602739726027 - expected_acc_and_f1 = 0.8751007252215954 - expected_loss = 0.412924896998732 + expected_acc = 0.84 + expected_f1 = 0.88 + expected_loss = 0.40 - results_per_api = dict() - for use_new_api in [True, False]: - results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=True, use_new_api=use_new_api) - assert_allclose(results['acc'], expected_acc, rtol=self.rtol) - assert_allclose(results['f1'], expected_f1, rtol=self.rtol) - assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol) - assert_allclose(results['loss'], expected_loss, rtol=self.rtol) - results_per_api[use_new_api] = results + results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=True) - verify_old_and_new_api_are_equal(results_per_api) + assert(results['acc'] >= expected_acc) + assert(results['f1'] >= expected_f1) + assert(results['loss'] <= expected_loss) def model_to_desc(self, model_name, model): if model_name.startswith('bert') or model_name.startswith('xlnet'): - new_model_desc = { + model_desc = { 'inputs': [ ('input_ids', ['batch', 'max_seq_len_in_batch'],), ('attention_mask', ['batch', 'max_seq_len_in_batch'],), @@ -168,33 +142,20 @@ class ORTGlueTest(unittest.TestCase): ('labels', ['batch', ],)], 'outputs': [('loss', [], True), ('logits', ['batch', 2])]} - model_desc = ModelDescription([ - IODescription('input_ids', ['batch', 'max_seq_len_in_batch']), - IODescription('attention_mask', ['batch', 'max_seq_len_in_batch']), - IODescription('token_type_ids', ['batch', 'max_seq_len_in_batch']), - IODescription('labels', ['batch',])], [ - IODescription('loss', []), - IODescription('logits', ['batch', 2])]) elif model_name.startswith('roberta'): - new_model_desc = { + model_desc = { 'inputs': [ ('input_ids', ['batch', 'max_seq_len_in_batch'],), ('attention_mask', ['batch', 'max_seq_len_in_batch'],), ('labels', ['batch', ],)], 'outputs': [('loss', [], True), ('logits', ['batch', 2])]} - model_desc = ModelDescription([ - IODescription('input_ids', ['batch', 'max_seq_len_in_batch']), - IODescription('attention_mask', ['batch', 'max_seq_len_in_batch']), - IODescription('labels', ['batch',])], [ - IODescription('loss', []), - IODescription('logits', ['batch', 2])]) else: raise RuntimeError("unsupported base model name {}.".format(model_name)) - return model_desc, new_model_desc + return model_desc - def run_glue(self, model_name, task_name, fp16, use_new_api): + def run_glue(self, model_name, task_name, fp16): model_args = ModelArguments(model_name_or_path=model_name, cache_dir=self.cache_dir) data_args = GlueDataTrainingArguments( task_name=task_name, data_dir=os.path.join(self.data_dir, task_name), @@ -270,17 +231,15 @@ class ORTGlueTest(unittest.TestCase): preds = np.squeeze(p.predictions) return glue_compute_metrics(data_args.task_name, preds, p.label_ids) - model_desc, new_model_desc = self.model_to_desc(model_name, model) + model_desc = self.model_to_desc(model_name, model) # Initialize the ORTTrainer within ORTTransformerTrainer trainer = ORTTransformerTrainer( model=model, model_desc=model_desc, - new_model_desc=new_model_desc, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=compute_metrics, - use_new_api=use_new_api, world_size=self.world_size, ) @@ -305,8 +264,13 @@ class ORTGlueTest(unittest.TestCase): return results if __name__ == "__main__": - local_rank = get_mpi_context_local_rank() - world_size = get_mpi_context_world_size() + if has_get_mpi_context_internal_api: + local_rank = get_mpi_context_local_rank() + world_size = get_mpi_context_world_size() + else: + local_rank = -1 + world_size = 1 + if world_size > 1: # mpi launch logger.warning("mpirun launch, local_rank / world_size: %s : % s", local_rank, world_size) diff --git a/orttraining/orttraining/test/python/orttraining_run_multiple_choice.py b/orttraining/orttraining/test/python/orttraining_run_multiple_choice.py index 0a6829ea6e..c1e8ebca95 100644 --- a/orttraining/orttraining/test/python/orttraining_run_multiple_choice.py +++ b/orttraining/orttraining/test/python/orttraining_run_multiple_choice.py @@ -92,34 +92,25 @@ class ORTMultipleChoiceTest(unittest.TestCase): self.logging_steps = 10 def test_bert_with_swag(self): - expected_acc = 0.7640207937618715 - expected_loss = 0.6234657892213054 - results_per_api = dict() - for use_new_api in [False, True]: - results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=False, use_new_api=use_new_api) - # assert_allclose(results['acc'], expected_acc) - # assert_allclose(results['loss'], expected_loss) - results_per_api[use_new_api] = results + expected_acc = 0.75 + expected_loss = 0.64 - verify_old_and_new_api_are_equal(results_per_api) + results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=False) + assert(results['acc'] >= expected_acc) + assert(results['loss'] <= expected_loss) def test_bert_fp16_with_swag(self): # larger batch can be handled with mixed precision self.train_batch_size = 32 - expected_acc = 0.7482255323402979 - expected_loss = 0.6665752871014349 + expected_acc = 0.73 + expected_loss = 0.68 - results_per_api = dict() - for use_new_api in [False, True]: - results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=True, use_new_api=use_new_api) - assert_allclose(results['acc'], expected_acc) - assert_allclose(results['loss'], expected_loss) - results_per_api[use_new_api] = results + results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=True) + assert(results['acc'] >= expected_acc) + assert(results['loss'] <= expected_loss) - verify_old_and_new_api_are_equal(results_per_api) - - def run_multiple_choice(self, model_name, task_name, fp16, use_new_api): + def run_multiple_choice(self, model_name, task_name, fp16): model_args = ModelArguments(model_name_or_path=model_name, cache_dir=self.cache_dir) data_args = DataTrainingArguments(task_name=task_name, data_dir=self.data_dir, max_seq_length=self.max_seq_length) @@ -209,14 +200,7 @@ class ORTMultipleChoiceTest(unittest.TestCase): return {"acc": simple_accuracy(preds, p.label_ids)} if model_name.startswith('bert'): - model_desc = ModelDescription([ - IODescription('input_ids', ['batch', num_labels, 'max_seq_len_in_batch']), - IODescription('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch']), - IODescription('token_type_ids', ['batch', num_labels, 'max_seq_len_in_batch']), - IODescription('labels', ['batch', num_labels])], [ - IODescription('loss', []), - IODescription('reshaped_logits', ['batch', num_labels])]) - new_model_desc = { + model_desc = { 'inputs': [ ('input_ids', ['batch', num_labels, 'max_seq_len_in_batch'],), ('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch'],), @@ -225,13 +209,7 @@ class ORTMultipleChoiceTest(unittest.TestCase): 'outputs': [('loss', [], True), ('reshaped_logits', ['batch', num_labels])]} else: - model_desc = ModelDescription([ - IODescription('input_ids', ['batch', num_labels, 'max_seq_len_in_batch']), - IODescription('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch']), - IODescription('labels', ['batch', num_labels])], [ - IODescription('loss', []), - IODescription('reshaped_logits', ['batch', num_labels])]) - new_model_desc = { + model_desc = { 'inputs': [ ('input_ids', ['batch', num_labels, 'max_seq_len_in_batch'],), ('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch'],), @@ -243,12 +221,10 @@ class ORTMultipleChoiceTest(unittest.TestCase): trainer = ORTTransformerTrainer( model=model, model_desc=model_desc, - new_model_desc=new_model_desc, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset, compute_metrics=compute_metrics, - use_new_api=use_new_api ) # Training diff --git a/orttraining/orttraining/test/python/orttraining_transformer_trainer.py b/orttraining/orttraining/test/python/orttraining_transformer_trainer.py index b2228bff28..ada4e573d5 100644 --- a/orttraining/orttraining/test/python/orttraining_transformer_trainer.py +++ b/orttraining/orttraining/test/python/orttraining_transformer_trainer.py @@ -100,21 +100,18 @@ class ORTTransformerTrainer: def __init__( self, model: PreTrainedModel, - model_desc: ModelDescription, - new_model_desc: dict, + model_desc: dict, args: TrainingArguments, train_dataset: Dataset, eval_dataset: Dataset, compute_metrics: Callable[[EvalPrediction], Dict], - world_size: Optional[int] = 1, - use_new_api : Optional[bool] = False, + world_size: Optional[int] = 1 ): """ """ self.model = model self.model_desc = model_desc - self.new_model_desc = new_model_desc self.args = args self.world_size = world_size self.data_collator = DefaultDataCollator() @@ -126,8 +123,6 @@ class ORTTransformerTrainer: if self.args.local_rank in [-1, 0]: os.makedirs(self.args.output_dir, exist_ok=True) - self.use_new_api = use_new_api - def get_train_dataloader(self) -> DataLoader: if self.train_dataset is None: raise ValueError("Trainer: training requires a train_dataset.") @@ -174,67 +169,41 @@ class ORTTransformerTrainer: t_total = int(len(train_dataloader) // self.args.gradient_accumulation_steps * self.args.num_train_epochs) num_train_epochs = self.args.num_train_epochs - if self.use_new_api: - lr_scheduler = orttrainer.optim.LinearWarmupLRScheduler(t_total, self.args.warmup_steps/float(t_total)) + lr_scheduler = orttrainer.optim.LinearWarmupLRScheduler(t_total, self.args.warmup_steps/float(t_total)) - loss_scaler = amp.DynamicLossScaler() if self.args.fp16 else None - device = self.args.device.type + loss_scaler = amp.DynamicLossScaler() if self.args.fp16 else None + device = self.args.device.type - device = f'{device}:{self.args.device.index}' if self.args.device.index else f'{device}:0' - options = orttrainer.ORTTrainerOptions({'batch' : { - 'gradient_accumulation_steps' : self.args.gradient_accumulation_steps}, - 'device': {'id': device}, - 'mixed_precision': { - 'enabled': self.args.fp16, - 'loss_scaler': loss_scaler}, - 'debug': {'deterministic_compute': True, }, - 'utils': { - 'grad_norm_clip': False}, - 'distributed': { - # we are running single node multi gpu test. thus world_rank = local_rank - # and world_size = self.args.n_gpu - 'world_rank': max(0, self.args.local_rank), - 'world_size': int(self.world_size), - 'local_rank': max(0, self.args.local_rank), - 'allreduce_post_accumulation': True}, - 'lr_scheduler': lr_scheduler - }) + device = f'{device}:{self.args.device.index}' if self.args.device.index else f'{device}:0' + options = orttrainer.ORTTrainerOptions({'batch' : { + 'gradient_accumulation_steps' : self.args.gradient_accumulation_steps}, + 'device': {'id': device}, + 'mixed_precision': { + 'enabled': self.args.fp16, + 'loss_scaler': loss_scaler}, + 'debug': {'deterministic_compute': True, }, + 'utils': { + 'grad_norm_clip': False}, + 'distributed': { + # we are running single node multi gpu test. thus world_rank = local_rank + # and world_size = self.args.n_gpu + 'world_rank': max(0, self.args.local_rank), + 'world_size': int(self.world_size), + 'local_rank': max(0, self.args.local_rank), + 'allreduce_post_accumulation': True}, + 'lr_scheduler': lr_scheduler + }) - param_optimizer = list(self.model.named_parameters()) - params = [{ - 'params': [n for n, p in param_optimizer if "bias" in n or "LayerNorm.weight" in n], - "weight_decay_mode": 1, }, { - 'params': [n for n, p in param_optimizer if not ("bias" in n or "LayerNorm.weight" in n)], - "weight_decay_mode": 1, } - ] + param_optimizer = list(self.model.named_parameters()) + params = [{ + 'params': [n for n, p in param_optimizer if "bias" in n or "LayerNorm.weight" in n], + "weight_decay_mode": 1, }, { + 'params': [n for n, p in param_optimizer if not ("bias" in n or "LayerNorm.weight" in n)], + "weight_decay_mode": 1, } + ] - optim_config = optim.AdamConfig(params=params, lr=2e-5, do_bias_correction=True) - self.model = orttrainer.ORTTrainer(self.model, self.new_model_desc, optim_config, options=options) - else: - def map_optimizer_attributes(name): - no_decay = "bias" in name or "LayerNorm.weight" in name - if no_decay: - return {"weight_decay_mode" : 1} - else: - return {"weight_decay_mode" : 1} - get_lr_this_step = get_linear_schedule_with_warmup(self.args.warmup_steps, t_total, self.args.learning_rate) - loss_scaler = LossScaler('loss_scale_input_name', True, up_scale_window=2000) if self.args.fp16 else None - self.model = ORTTrainer(self.model, None, - self.model_desc, - "AdamOptimizer", - map_optimizer_attributes=map_optimizer_attributes, - learning_rate_description=IODescription('Learning_Rate', [1,], torch.float32), - device=self.args.device, - gradient_accumulation_steps=self.args.gradient_accumulation_steps, - world_rank=max(0, self.args.local_rank), - world_size=int(self.world_size), - use_mixed_precision=self.args.fp16, - allreduce_post_accumulation=True, - get_lr_this_step=get_lr_this_step, - loss_scaler=loss_scaler, - enable_grad_norm_clip=False, - _opset_version=12, - _use_deterministic_compute=True) + optim_config = optim.AdamConfig(params=params, lr=2e-5, do_bias_correction=True) + self.model = orttrainer.ORTTrainer(self.model, self.model_desc, optim_config, options=options) # Train! logger.info("***** Running training *****") @@ -289,9 +258,7 @@ class ORTTransformerTrainer: logs[eval_key] = value loss_scalar = (tr_loss - logging_loss) / self.args.logging_steps - if not self.use_new_api: - learning_rate_scalar = get_lr_this_step(global_step) - logs["learning_rate"] = learning_rate_scalar + logs["loss"] = loss_scalar logging_loss = tr_loss @@ -362,9 +329,6 @@ class ORTTransformerTrainer: preds: np.ndarray = None label_ids: np.ndarray = None - if not self.use_new_api: - self.model.eval() - for inputs in tqdm(dataloader, desc=description): has_labels = any(inputs.get(k) is not None for k in ["labels", "masked_lm_labels"]) @@ -372,10 +336,8 @@ class ORTTransformerTrainer: inputs[k] = v.to(self.args.device) with torch.no_grad(): - if self.use_new_api: - outputs = self.model.eval_step(**inputs) - else: - outputs = self.model(**inputs) + outputs = self.model.eval_step(**inputs) + if has_labels: step_eval_loss, logits = outputs[:2] eval_losses += [step_eval_loss.mean().item()] diff --git a/orttraining/tools/ci_test/download_e2e_test_data.py b/orttraining/tools/ci_test/download_e2e_test_data.py index 0978145ebf..324f97b4a8 100755 --- a/orttraining/tools/ci_test/download_e2e_test_data.py +++ b/orttraining/tools/ci_test/download_e2e_test_data.py @@ -19,10 +19,6 @@ sys.path.append(os.path.join(REPO_DIR, "tools", "python")) import get_azcopy # noqa: E402 -# update these if the E2E test data changes -ARCHIVE_BLOB_URL = "https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z" -ARCHIVE_SHA256_DIGEST = "B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9" - def _download(azcopy_path, url, local_path): subprocess.run([azcopy_path, "cp", "--log-level", "NONE", url, local_path], check=True) @@ -47,15 +43,18 @@ def _check_file_sha256_digest(path, expected_digest): def main(): parser = argparse.ArgumentParser(description="Downloads training end-to-end test data.") - parser.add_argument("target_dir", help="The test data destination directory.") + parser.add_argument("--azure_blob_url", required=True, help="The test data destination directory.") + parser.add_argument("--target_dir", required=True, help="The test data destination directory.") + parser.add_argument("--archive_sha256_digest", help="The test data destination directory.") args = parser.parse_args() with tempfile.TemporaryDirectory() as temp_dir, \ get_azcopy.get_azcopy() as azcopy_path: archive_path = os.path.join(temp_dir, "archive.zip") - print("Downloading E2E test data from '{}'...".format(ARCHIVE_BLOB_URL)) - _download(azcopy_path, ARCHIVE_BLOB_URL, archive_path) - _check_file_sha256_digest(archive_path, ARCHIVE_SHA256_DIGEST) + print("Downloading E2E test data from '{}'...".format(args.azure_blob_url)) + _download(azcopy_path, args.azure_blob_url, archive_path) + if args.archive_sha256_digest: + _check_file_sha256_digest(archive_path, args.archive_sha256_digest) print("Extracting to '{}'...".format(args.target_dir)) shutil.unpack_archive(archive_path, args.target_dir) print("Done.") diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 291dad6acd..d02cf6f56c 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1196,13 +1196,6 @@ def run_training_python_frontend_e2e_tests(cwd): 'mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', sys.executable, bert_pretrain_script, 'ORTBertPretrainTest.test_pretrain_convergence'], cwd=cwd) - # a long run - log.debug('RUN: mpirun -n {} ''-x' 'NCCL_DEBUG=INFO'' {} {}'.format( - ngpus, sys.executable, bert_pretrain_script)) - run_subprocess([ - 'mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', sys.executable, - bert_pretrain_script], cwd=cwd) - log.debug('RUN: mpirun -n {} {} orttraining_run_glue.py'.format(ngpus, sys.executable)) run_subprocess([ 'mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', sys.executable, 'orttraining_run_glue.py'], cwd=cwd) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml index d4a85abf47..b316eb472e 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-ci-pipeline.yml @@ -10,8 +10,12 @@ jobs: clean: true submodules: recursive + # update these if the E2E test data changes - script: | - orttraining/tools/ci_test/download_e2e_test_data.py $(Build.BinariesDirectory)/training_e2e_test_data + orttraining/tools/ci_test/download_e2e_test_data.py \ + --azure_blob_url https://onnxruntimetestdata.blob.core.windows.net/training/onnxruntime_training_data.zip?snapshot=2020-06-15T23:17:35.8314853Z \ + --target_dir $(Build.BinariesDirectory)/training_e2e_test_data \ + --archive_sha256_digest B01C169B6550D1A0A6F1B4E2F34AE2A8714B52DBB70AC04DA85D371F691BDFF9 displayName: 'Download training end-to-end test data' - script: | diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml index 54351005e5..9769337dee 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml @@ -26,8 +26,6 @@ jobs: continueOnError: true condition: always() - # insert a python frontend test data preparation step here - - script: > tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) diff --git a/tools/ci_build/github/linux/docker/scripts/training/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/requirements.txt index 926f0ceff5..5668c537cd 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/requirements.txt @@ -1,10 +1,10 @@ --pre --f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html +-f https://download.pytorch.org/whl/torch_stable.html # transformers requires sklearn sklearn transformers==v2.10.0 -torch==1.6.0.dev20200610 -torchvision==0.7.0.dev20200610 -torchtext==0.6.0.dev20200610 +torch==1.6.0+cu101 +torchvision==0.7.0+cu101 +torchtext==0.7.0 tensorboard==v2.0.0 h5py