mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
matching multiple choice between new and old apis (#4918)
* matching multiple choice between new and old apis * update according to reviewer's comments Co-authored-by: liqun <liqun@OrtTrainingDev4.af05slrtruoetgaxwwjv5nsq5e.px.internal.cloudapp.net>
This commit is contained in:
liqunfu
GitHub
parent
0d3bbfdd0f
commit
b3783a9f85
3 changed files with 90 additions and 36 deletions
|
|
@ -33,6 +33,12 @@ import torch
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def verify_old_and_new_api_are_equal(results_per_api):
|
||||
new_api_results = results_per_api[True]
|
||||
old_api_results = results_per_api[False]
|
||||
for key in new_api_results.keys():
|
||||
assert_allclose(new_api_results[key], old_api_results[key])
|
||||
|
||||
@dataclass
|
||||
class ModelArguments:
|
||||
"""
|
||||
|
|
@ -75,12 +81,16 @@ class ORTGlueTest(unittest.TestCase):
|
|||
expected_acc_and_f1 = 0.9013897443794272
|
||||
expected_loss = 0.35917433314755853
|
||||
|
||||
results_per_api = dict()
|
||||
for use_new_api in [True, False]:
|
||||
results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=False, use_new_api=use_new_api)
|
||||
assert_allclose(results['acc'], expected_acc, rtol=self.rtol)
|
||||
assert_allclose(results['f1'], expected_f1, rtol=self.rtol)
|
||||
assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol)
|
||||
assert_allclose(results['loss'], expected_loss, rtol=self.rtol)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
|
||||
def test_roberta_fp16_with_mrpc(self):
|
||||
expected_acc = 0.8946078431372549
|
||||
|
|
@ -88,12 +98,16 @@ class ORTGlueTest(unittest.TestCase):
|
|||
expected_acc_and_f1 = 0.90965068163868
|
||||
expected_loss = 0.3052181116506165
|
||||
|
||||
results_per_api = dict()
|
||||
for use_new_api in [True, False]:
|
||||
results = self.run_glue(model_name="roberta-base", task_name="MRPC", fp16=True, use_new_api=use_new_api)
|
||||
assert_allclose(results['acc'], expected_acc, rtol=self.rtol)
|
||||
assert_allclose(results['f1'], expected_f1, rtol=self.rtol)
|
||||
assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol)
|
||||
assert_allclose(results['loss'], expected_loss, rtol=self.rtol)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
|
||||
def test_bert_with_mrpc(self):
|
||||
if self.local_rank == -1:
|
||||
|
|
@ -109,8 +123,12 @@ class ORTGlueTest(unittest.TestCase):
|
|||
|
||||
if self.local_rank == -1:
|
||||
# not parallel case, we can run both new and old api tests
|
||||
results_per_api = dict()
|
||||
for use_new_api in [True, False]:
|
||||
results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=False, use_new_api=use_new_api)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
else:
|
||||
# with parallel training, TrainingArguments can only be created once (due to its cached _setup_devices)
|
||||
# thus we can only choose one test case to run.
|
||||
|
|
@ -128,12 +146,16 @@ class ORTGlueTest(unittest.TestCase):
|
|||
expected_acc_and_f1 = 0.8751007252215954
|
||||
expected_loss = 0.412924896998732
|
||||
|
||||
results_per_api = dict()
|
||||
for use_new_api in [True, False]:
|
||||
results = self.run_glue(model_name="bert-base-cased", task_name="MRPC", fp16=True, use_new_api=use_new_api)
|
||||
assert_allclose(results['acc'], expected_acc, rtol=self.rtol)
|
||||
assert_allclose(results['f1'], expected_f1, rtol=self.rtol)
|
||||
assert_allclose(results['acc_and_f1'], expected_acc_and_f1, rtol=self.rtol)
|
||||
assert_allclose(results['loss'], expected_loss, rtol=self.rtol)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
|
||||
def model_to_desc(self, model_name, model):
|
||||
if model_name.startswith('bert') or model_name.startswith('xlnet'):
|
||||
|
|
@ -146,12 +168,12 @@ class ORTGlueTest(unittest.TestCase):
|
|||
'outputs': [('loss', [], True),
|
||||
('logits', ['batch', 2])]}
|
||||
model_desc = ModelDescription([
|
||||
IODescription('input_ids', ['batch', 'max_seq_len_in_batch'], torch.int64, num_classes=model.config.vocab_size),
|
||||
IODescription('attention_mask', ['batch', 'max_seq_len_in_batch'], torch.int64, num_classes=2),
|
||||
IODescription('token_type_ids', ['batch', 'max_seq_len_in_batch'], torch.int64, num_classes=2),
|
||||
IODescription('labels', ['batch',], torch.int64, num_classes=2)], [
|
||||
IODescription('loss', [], torch.float32),
|
||||
IODescription('logits', ['batch', 2], torch.float32)])
|
||||
IODescription('input_ids', ['batch', 'max_seq_len_in_batch']),
|
||||
IODescription('attention_mask', ['batch', 'max_seq_len_in_batch']),
|
||||
IODescription('token_type_ids', ['batch', 'max_seq_len_in_batch']),
|
||||
IODescription('labels', ['batch',])], [
|
||||
IODescription('loss', []),
|
||||
IODescription('logits', ['batch', 2])])
|
||||
elif model_name.startswith('roberta'):
|
||||
new_model_desc = {
|
||||
'inputs': [
|
||||
|
|
@ -161,11 +183,11 @@ class ORTGlueTest(unittest.TestCase):
|
|||
'outputs': [('loss', [], True),
|
||||
('logits', ['batch', 2])]}
|
||||
model_desc = ModelDescription([
|
||||
IODescription('input_ids', ['batch', 'max_seq_len_in_batch'], torch.int64, num_classes=model.config.vocab_size),
|
||||
IODescription('attention_mask', ['batch', 'max_seq_len_in_batch'], torch.int64, num_classes=2),
|
||||
IODescription('labels', ['batch',], torch.int64, num_classes=2)], [
|
||||
IODescription('loss', [], torch.float32),
|
||||
IODescription('logits', ['batch', 2], torch.float32)])
|
||||
IODescription('input_ids', ['batch', 'max_seq_len_in_batch']),
|
||||
IODescription('attention_mask', ['batch', 'max_seq_len_in_batch']),
|
||||
IODescription('labels', ['batch',])], [
|
||||
IODescription('loss', []),
|
||||
IODescription('logits', ['batch', 2])])
|
||||
else:
|
||||
raise RuntimeError("unsupported base model name {}.".format(model_name))
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from orttraining_transformer_trainer import ORTTransformerTrainer
|
|||
import torch
|
||||
|
||||
from utils_multiple_choice import MultipleChoiceDataset, Split, SwagProcessor
|
||||
from orttraining_run_glue import verify_old_and_new_api_are_equal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -78,7 +79,7 @@ class ORTMultipleChoiceTest(unittest.TestCase):
|
|||
def setUp(self):
|
||||
# configurations not to be changed accoss tests
|
||||
self.max_seq_length = 80
|
||||
self.train_batch_size = 2
|
||||
self.train_batch_size = 16
|
||||
self.eval_batch_size = 2
|
||||
self.learning_rate = 2e-5
|
||||
self.num_train_epochs = 1.0
|
||||
|
|
@ -91,22 +92,34 @@ class ORTMultipleChoiceTest(unittest.TestCase):
|
|||
self.logging_steps = 10
|
||||
|
||||
def test_bert_with_swag(self):
|
||||
expected_acc = 0.7752174347695692
|
||||
expected_loss = 0.5848011111033019
|
||||
expected_acc = 0.7640207937618715
|
||||
expected_loss = 0.6234657892213054
|
||||
results_per_api = dict()
|
||||
for use_new_api in [False, True]:
|
||||
results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=False, use_new_api=use_new_api)
|
||||
# assert_allclose(results['acc'], expected_acc)
|
||||
# assert_allclose(results['loss'], expected_loss)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=False)
|
||||
assert_allclose(results['acc'], expected_acc)
|
||||
assert_allclose(results['loss'], expected_loss)
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
|
||||
def test_bert_fp16_with_swag(self):
|
||||
expected_acc = 0.7765170448865341
|
||||
expected_loss = 0.5851960482903881
|
||||
# larger batch can be handled with mixed precision
|
||||
self.train_batch_size = 32
|
||||
|
||||
results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=True)
|
||||
assert_allclose(results['acc'], expected_acc)
|
||||
assert_allclose(results['loss'], expected_loss)
|
||||
expected_acc = 0.7482255323402979
|
||||
expected_loss = 0.6665529619455844
|
||||
|
||||
def run_multiple_choice(self, model_name, task_name, fp16):
|
||||
results_per_api = dict()
|
||||
for use_new_api in [False, True]:
|
||||
results = self.run_multiple_choice(model_name="bert-base-cased", task_name="swag", fp16=True, use_new_api=use_new_api)
|
||||
assert_allclose(results['acc'], expected_acc)
|
||||
assert_allclose(results['loss'], expected_loss)
|
||||
results_per_api[use_new_api] = results
|
||||
|
||||
verify_old_and_new_api_are_equal(results_per_api)
|
||||
|
||||
def run_multiple_choice(self, model_name, task_name, fp16, use_new_api):
|
||||
model_args = ModelArguments(model_name_or_path=model_name, cache_dir=self.cache_dir)
|
||||
data_args = DataTrainingArguments(task_name=task_name, data_dir=self.data_dir,
|
||||
max_seq_length=self.max_seq_length)
|
||||
|
|
@ -150,6 +163,7 @@ class ORTMultipleChoiceTest(unittest.TestCase):
|
|||
finetuning_task=data_args.task_name,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
|
||||
cache_dir=model_args.cache_dir,
|
||||
|
|
@ -196,29 +210,45 @@ class ORTMultipleChoiceTest(unittest.TestCase):
|
|||
|
||||
if model_name.startswith('bert'):
|
||||
model_desc = ModelDescription([
|
||||
IODescription('input_ids', [self.train_batch_size, num_labels, data_args.max_seq_length], torch.int64, num_classes=model.config.vocab_size),
|
||||
IODescription('attention_mask', [self.train_batch_size, num_labels, data_args.max_seq_length], torch.int64, num_classes=2),
|
||||
IODescription('token_type_ids', [self.train_batch_size, num_labels, data_args.max_seq_length], torch.int64, num_classes=2),
|
||||
IODescription('labels', [self.train_batch_size, num_labels], torch.int64, num_classes=num_labels)], [
|
||||
IODescription('loss', [], torch.float32),
|
||||
IODescription('reshaped_logits', [self.train_batch_size, num_labels], torch.float32)])
|
||||
IODescription('input_ids', ['batch', num_labels, 'max_seq_len_in_batch']),
|
||||
IODescription('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch']),
|
||||
IODescription('token_type_ids', ['batch', num_labels, 'max_seq_len_in_batch']),
|
||||
IODescription('labels', ['batch', num_labels])], [
|
||||
IODescription('loss', []),
|
||||
IODescription('reshaped_logits', ['batch', num_labels])])
|
||||
new_model_desc = {
|
||||
'inputs': [
|
||||
('input_ids', ['batch', num_labels, 'max_seq_len_in_batch'],),
|
||||
('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch'],),
|
||||
('token_type_ids', ['batch', num_labels, 'max_seq_len_in_batch'],),
|
||||
('labels', ['batch', num_labels],)],
|
||||
'outputs': [('loss', [], True),
|
||||
('reshaped_logits', ['batch', num_labels])]}
|
||||
else:
|
||||
model_desc = ModelDescription([
|
||||
IODescription('input_ids', ['batch', num_labels, 'max_seq_len_in_batch'], torch.int64, num_classes=model.config.vocab_size),
|
||||
IODescription('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch'], torch.int64, num_classes=2),
|
||||
IODescription('labels', ['batch', num_labels], torch.int64, num_classes=num_labels)], [
|
||||
IODescription('loss', [], torch.float32),
|
||||
IODescription('reshaped_logits', ['batch', num_labels], torch.float32)])
|
||||
IODescription('input_ids', ['batch', num_labels, 'max_seq_len_in_batch']),
|
||||
IODescription('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch']),
|
||||
IODescription('labels', ['batch', num_labels])], [
|
||||
IODescription('loss', []),
|
||||
IODescription('reshaped_logits', ['batch', num_labels])])
|
||||
new_model_desc = {
|
||||
'inputs': [
|
||||
('input_ids', ['batch', num_labels, 'max_seq_len_in_batch'],),
|
||||
('attention_mask', ['batch', num_labels, 'max_seq_len_in_batch'],),
|
||||
('labels', ['batch', num_labels],)],
|
||||
'outputs': [('loss', [], True),
|
||||
('reshaped_logits', ['batch', num_labels])]}
|
||||
|
||||
# Initialize the ORTTrainer within ORTTransformerTrainer
|
||||
trainer = ORTTransformerTrainer(
|
||||
model=model,
|
||||
model_desc=model_desc,
|
||||
new_model_desc=None,
|
||||
new_model_desc=new_model_desc,
|
||||
args=training_args,
|
||||
train_dataset=train_dataset,
|
||||
eval_dataset=eval_dataset,
|
||||
compute_metrics=compute_metrics,
|
||||
use_new_api=use_new_api
|
||||
)
|
||||
|
||||
# Training
|
||||
|
|
|
|||
|
|
@ -178,7 +178,9 @@ class ORTTransformerTrainer:
|
|||
loss_scaler = amp.DynamicLossScaler() if self.args.fp16 else None
|
||||
device = self.args.device.type
|
||||
device = f'{device}:{self.args.device.index}' if self.args.device.index else f'{device}:0'
|
||||
options = orttrainer.ORTTrainerOptions({'device': {'id': device},
|
||||
options = orttrainer.ORTTrainerOptions({'batch' : {
|
||||
'gradient_accumulation_steps' : self.args.gradient_accumulation_steps},
|
||||
'device': {'id': device},
|
||||
'mixed_precision': {
|
||||
'enabled': self.args.fp16,
|
||||
'loss_scaler': loss_scaler},
|
||||
|
|
|
|||
Loading…
Reference in a new issue