saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix frontend bug.

Browse source 
old ort session may already exists when creating new ort session, this may cause OOM error
This commit is contained in:
zhijxu 2020-11-11 11:41:38 +08:00 committed by zhijxu-MS
parent c4818d36ed
commit 89902c2519
2 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
orttraining/orttraining/python/ort_trainer.py
View file

@ -696,6 +696,9 @@ class ORTTrainer():
if self.run_symbolic_shape_infer:
self.onnx_model_ = SymbolicShapeInference.infer_shapes(self.onnx_model_, auto_merge=True, guess_output_rank=True)
# old ort session may already exists and occupies GPU memory when creating new session, this may cause OOM error.
# for example, load_state_dict will be called before returing the function, and it calls _init_session again
del self.session
self.session, self.train_io_binding, self.eval_io_binding, self.output_name, _, self.output_types = \
create_ort_training_session_with_optimizer(
self.onnx_model_, self.device_,

3
orttraining/orttraining/python/training/orttrainer.py
View file

@ -685,6 +685,9 @@ class ORTTrainer(object):
if self.options.utils.run_symbolic_shape_infer:
self._onnx_model = SymbolicShapeInference.infer_shapes(self._onnx_model, auto_merge=True, guess_output_rank=True)
# old ort session may already exists and occupies GPU memory when creating new session, this may cause OOM error.
# for example, load_state_dict will be called before returing the function, and it calls _init_session again
del self._training_session
# Create training session used by train_step
self._create_ort_training_session()