diff --git a/onnxruntime/core/graph/graph_flatbuffers_utils.h b/onnxruntime/core/graph/graph_flatbuffers_utils.h index 9804c3ba50..fefbbfca6a 100644 --- a/onnxruntime/core/graph/graph_flatbuffers_utils.h +++ b/onnxruntime/core/graph/graph_flatbuffers_utils.h @@ -22,7 +22,7 @@ class Node; class Path; namespace logging { - class Logger; +class Logger; } namespace experimental { diff --git a/orttraining/orttraining/core/optimizer/bias_dropout_fusion.cc b/onnxruntime/core/optimizer/bias_dropout_fusion.cc similarity index 99% rename from orttraining/orttraining/core/optimizer/bias_dropout_fusion.cc rename to onnxruntime/core/optimizer/bias_dropout_fusion.cc index 6baa3e6b12..442a9ae88a 100644 --- a/orttraining/orttraining/core/optimizer/bias_dropout_fusion.cc +++ b/onnxruntime/core/optimizer/bias_dropout_fusion.cc @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "core/optimizer/initializer.h" -#include "orttraining/core/optimizer/bias_dropout_fusion.h" +#include "core/optimizer/bias_dropout_fusion.h" #include "core/graph/graph_utils.h" #include diff --git a/orttraining/orttraining/core/optimizer/bias_dropout_fusion.h b/onnxruntime/core/optimizer/bias_dropout_fusion.h similarity index 100% rename from orttraining/orttraining/core/optimizer/bias_dropout_fusion.h rename to onnxruntime/core/optimizer/bias_dropout_fusion.h diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 7463c598ee..a4f138eb58 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -39,7 +39,7 @@ #include "core/optimizer/unsqueeze_elimination.h" #include "core/session/onnxruntime_session_options_config_keys.h" #include "core/optimizer/matmul_transpose_fusion.h" -#include "orttraining/core/optimizer/bias_dropout_fusion.h" +#include "core/optimizer/bias_dropout_fusion.h" namespace onnxruntime { class IExecutionProvider; diff --git a/onnxruntime/core/providers/cpu/nn/pool_base.h b/onnxruntime/core/providers/cpu/nn/pool_base.h index f4d0082866..3d1edfbdaf 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_base.h +++ b/onnxruntime/core/providers/cpu/nn/pool_base.h @@ -88,7 +88,7 @@ class LpPool { template static void Process(const T& x_data, T& y_data, const PoolProcessContext& cxt) { - y_data += static_cast(std::pow(x_data, cxt.p_)); + y_data += static_cast(std::pow(std::abs(x_data), cxt.p_)); } template diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index ba90d05a8a..782a46fc32 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -1961,7 +1961,7 @@ CUDAExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, // none of the provided registries has a CUDA kernel for this node if (cuda_kernel_def == nullptr) { - LOGS_DEFAULT(INFO) << "CUDA kernel not found in registries for Op type: " << node.OpType() << " node name: " << node.Name(); + LOGS_DEFAULT(WARNING) << "CUDA kernel not found in registries for Op type: " << node.OpType() << " node name: " << node.Name(); continue; } diff --git a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu index 3484daec37..66d50dafa5 100644 --- a/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu +++ b/onnxruntime/core/providers/cuda/math/unary_elementwise_ops_impl.cu @@ -139,12 +139,6 @@ void Impl_Cast( #define SPECIALIZED_CAST_IMPL2_BF16(T) #endif -#if CUDA_VERSION >= 11000 && (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) -#define SPECIALIZED_CAST_IMPL2_BF16(T) SPECIALIZED_CAST_IMPL2(T, nv_bfloat16) -#else -#define SPECIALIZED_CAST_IMPL2_BF16(T) -#endif - #define SPECIALIZED_CAST_FROM(T) \ SPECIALIZED_CAST_IMPL2(T, half) \ SPECIALIZED_CAST_IMPL2_BF16(T) \ diff --git a/onnxruntime/test/framework/tensorutils_test.cc b/onnxruntime/test/framework/tensorutils_test.cc index 537c8686a8..39f0f6d6f1 100644 --- a/onnxruntime/test/framework/tensorutils_test.cc +++ b/onnxruntime/test/framework/tensorutils_test.cc @@ -191,56 +191,6 @@ TEST(TensorProtoUtilsTest, UnpackTensorWithExternalData) { TestUnpackExternalTensor(TensorProto_DataType_BOOL, model_path); } -template -static void CreateTensorWithExternalData( - TensorProto_DataType type, - const std::vector& test_data, - std::basic_string& filename, - TensorProto& tensor_proto) { - // Create external data - FILE* fp; - CreateTestFile(fp, filename); - size_t size_in_bytes = test_data.size() * sizeof(T); - ASSERT_EQ(size_in_bytes, fwrite(test_data.data(), 1, size_in_bytes, fp)); - ASSERT_EQ(0, fclose(fp)); - - // set the tensor_proto to reference this external data - onnx::StringStringEntryProto* location = tensor_proto.mutable_external_data()->Add(); - location->set_key("location"); - location->set_value(ToMBString(filename)); - tensor_proto.mutable_dims()->Add(test_data.size()); - tensor_proto.set_data_location(onnx::TensorProto_DataLocation_EXTERNAL); - tensor_proto.set_data_type(type); -} - -template -static void TestUnpackExternalTensor(TensorProto_DataType type, const Path& model_path) { - // Create external data - std::basic_string filename(ORT_TSTR("tensor_XXXXXX")); - TensorProto tensor_proto; - auto test_data = CreateValues(); - CreateTensorWithExternalData(type, test_data, filename, tensor_proto); - std::unique_ptr file_deleter(const_cast(filename.c_str()), - DeleteFileFromDisk); - - // Unpack tensor with external data - std::vector val(test_data.size()); - auto st = utils::UnpackTensor(tensor_proto, model_path, val.data(), test_data.size()); - ASSERT_TRUE(st.IsOK()) << st.ErrorMessage(); - - // Validate data - for (size_t i = 0; i < test_data.size(); i++) { - ASSERT_EQ(val[i], test_data[i]); - } -} - -TEST(TensorProtoUtilsTest, UnpackTensorWithExternalData) { - Path model_path; - TestUnpackExternalTensor(TensorProto_DataType_FLOAT, model_path); - TestUnpackExternalTensor(TensorProto_DataType_DOUBLE, model_path); - TestUnpackExternalTensor(TensorProto_DataType_INT32, model_path); -} - template static NodeProto CreateConstantNode(const std::string& attrib_name, AttributeProto_AttributeType type, std::function add_data) { diff --git a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc index f84da5cf9a..b9db7ad47f 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc @@ -1264,25 +1264,6 @@ TEST(PoolTest, LpPool) { test.Run(); } -TEST(PoolTest, LpPoolWithNegativeNumbers) { - OpTester test("LpPool"); - - test.AddAttribute("p", static_cast(1)); - test.AddAttribute("auto_pad", ""); - test.AddAttribute("strides", std::vector{2}); - test.AddAttribute("pads", vector{0, 0}); - test.AddAttribute("kernel_shape", vector{2}); - - std::vector x_vals = {0.2f, -0.6f}; - std::vector x_dims = {1, 1, 2}; - std::vector expected_dims = {1, 1, 1}; - std::vector expected_vals = {-0.4f}; - - test.AddInput("X", x_dims, x_vals); - test.AddOutput("Y", expected_dims, expected_vals); - test.Run(); -} - TEST(PoolTest, GlobalLpPool) { OpTester test("GlobalLpPool"); test.AddAttribute("p", static_cast(3)); diff --git a/onnxruntime/test/python/quantization/test_calibration.py b/onnxruntime/test/python/quantization/test_calibration.py index b3b3af66a7..64441af294 100644 --- a/onnxruntime/test/python/quantization/test_calibration.py +++ b/onnxruntime/test/python/quantization/test_calibration.py @@ -11,50 +11,40 @@ import onnx import onnxruntime import numpy as np from onnx import helper, TensorProto, numpy_helper -from onnxruntime.quantization.calibrate import calibrate, CalibrationDataReader, ONNXCalibrater +from onnxruntime.quantization.calibrate import CalibrationDataReader, MinMaxCalibrater def generate_input_initializer(tensor_shape, tensor_dtype, input_name): ''' Helper function to generate initializers for test inputs ''' - tensor = np.random.ranf(tensor_shape).astype(tensor_dtype) + tensor = np.random.normal(0, 0.3, tensor_shape).astype(tensor_dtype) init = numpy_helper.from_array(tensor, input_name) return init class TestDataReader(CalibrationDataReader): - '''for test purpose''' - def __init__(self): - pass - - def get_next(self): - return None - - -class TestDataReaderSecond(CalibrationDataReader): '''for test purpose''' def __init__(self): self.preprocess_flag = True self.enum_data_dicts = [] + self.count = 4 + self.input_data_list = [] + for _ in range(self.count): + self.input_data_list.append(np.random.normal(0, 0.33, [1, 3, 1, 3]).astype(np.float32)) def get_next(self): if self.preprocess_flag: self.preprocess_flag = False - nhwc_data_list = [] - nhwc_data_list.append( - np.array([[[[0.45, 0.60, 0.75]], [[0.25, 0.50, 0.75]], [[0.90, 0.70, 0.50]]]]).astype(np.float32)) - nhwc_data_list.append( - np.array([[[[0.62, 0.94, 0.38]], [[0.70, 0.13, 0.07]], [[0.89, 0.75, 0.84]]]]).astype(np.float32)) - nhwc_data_list.append( - np.array([[[[0.64, 0.24, 0.97]], [[0.82, 0.58, 0.27]], [[0.019, 0.34, 0.02]]]]).astype(np.float32)) - input_name = 'input0' - self.enum_data_dicts = iter([{input_name: nhwc_data} for nhwc_data in nhwc_data_list]) + input_name = 'input' + self.enum_data_dicts = iter([{input_name: input_data} for input_data in self.input_data_list]) return next(self.enum_data_dicts, None) + def rewind(self): + self.preprocess_flag = True class TestCalibrate(unittest.TestCase): - def test_augment_graph(self): + def test_augment_graph_config_1(self): ''' TEST_CONFIG_1''' # Conv @@ -80,11 +70,9 @@ class TestCalibrate(unittest.TestCase): onnx.save(model, test_model_path) # Augmenting graph - data_reader = TestDataReader() augmented_model_path = './augmented_test_model_1.onnx' - calibrater = ONNXCalibrater(test_model_path, data_reader, ['Conv', 'MatMul'], [], [], augmented_model_path) - augmented_model = calibrater.augment_graph() - onnx.save(augmented_model, augmented_model_path) + calibrater = MinMaxCalibrater(test_model_path, ['Conv', 'MatMul'], augmented_model_path) + augmented_model = calibrater.get_augment_model() # Checking if each added ReduceMin and ReduceMax node and its output exists augmented_model_node_names = [node.name for node in augmented_model.graph.node] @@ -100,9 +88,8 @@ class TestCalibrate(unittest.TestCase): for output in added_outputs: self.assertTrue(output in augmented_model_outputs) - print('Finished TEST_CONFIG_1') + def test_augment_graph_config_2(self): '''TEST_CONFIG_2''' - # Conv # | # Conv @@ -112,11 +99,11 @@ class TestCalibrate(unittest.TestCase): J = helper.make_tensor_value_info('J', TensorProto.FLOAT, [1, 1, 3, 3]) K = helper.make_tensor_value_info('K', TensorProto.FLOAT, [1, 1, 5, 5]) conv_node_1 = onnx.helper.make_node('Conv', ['G', 'H'], ['I'], - name='Conv', + name='Conv1', kernel_shape=[3, 3], pads=[1, 1, 1, 1]) conv_node_2 = onnx.helper.make_node('Conv', ['I', 'J'], ['K'], - name='Conv', + name='Conv2', kernel_shape=[3, 3], pads=[1, 1, 1, 1]) graph = helper.make_graph([conv_node_1, conv_node_2], 'test_graph_2', [G, H, J], [K]) @@ -124,12 +111,9 @@ class TestCalibrate(unittest.TestCase): test_model_path = './test_model_2.onnx' onnx.save(model, test_model_path) - # Augmenting graph - data_reader = TestDataReader() augmented_model_path = './augmented_test_model_2.onnx' - calibrater = ONNXCalibrater(test_model_path, data_reader, ['Conv', 'MatMul'], [], [], augmented_model_path) - augmented_model = calibrater.augment_graph() - onnx.save(augmented_model, augmented_model_path) + calibrater = MinMaxCalibrater(test_model_path, ['Conv', 'MatMul'], augmented_model_path) + augmented_model = calibrater.get_augment_model() augmented_model_node_names = [node.name for node in augmented_model.graph.node] augmented_model_outputs = [output.name for output in augmented_model.graph.output] @@ -144,16 +128,20 @@ class TestCalibrate(unittest.TestCase): for output in added_outputs: self.assertTrue(output in augmented_model_outputs) - print('Finished TEST_CONFIG_2') + def test_augment_graph_config_3(self): '''TEST_CONFIG_3''' - # Relu - # | - # Conv \ - # | | - # Clip | - # | / - # MatMul + # (input) + # | + # Relu + # / \ + # Conv \ + # | | + # Clip | + # | / + # MatMul + # | + # (output) L = helper.make_tensor_value_info('L', TensorProto.FLOAT, [1, 1, 5, 5]) N = helper.make_tensor_value_info('N', TensorProto.FLOAT, [1, 1, 3, 3]) @@ -171,11 +159,9 @@ class TestCalibrate(unittest.TestCase): onnx.save(model, test_model_path) # Augmenting graph - data_reader = TestDataReader() augmented_model_path = './augmented_test_model_3.onnx' - calibrater = ONNXCalibrater(test_model_path, data_reader, ['Conv', 'MatMul'], [], [], augmented_model_path) - augmented_model = calibrater.augment_graph() - onnx.save(augmented_model, augmented_model_path) + calibrater = MinMaxCalibrater(test_model_path, ['Conv', 'MatMul'], augmented_model_path) + augmented_model = calibrater.get_augment_model() augmented_model_node_names = [node.name for node in augmented_model.graph.node] augmented_model_outputs = [output.name for output in augmented_model.graph.output] @@ -196,86 +182,81 @@ class TestCalibrate(unittest.TestCase): for output in added_outputs: self.assertTrue(output in augmented_model_outputs) - print('Finished TEST_CONFIG_3') - - def test_quant_param_calculation(self): - '''TEST_CONFIG_4''' - - # Relu - # | \ - # Conv \ - # | \ - # Relu | - # | Conv - # Conv / - # \ / - # | - # Add - - input0 = helper.make_tensor_value_info('input0', TensorProto.FLOAT, [1, 3, 1, 3]) - output = helper.make_tensor_value_info('output', TensorProto.FLOAT, [1, 3, 1, 3]) - - X1_weight = generate_input_initializer([3, 3, 1, 1], np.float32, 'X1_weight') - X1_bias = generate_input_initializer([3], np.float32, 'X1_bias') - X3_weight = generate_input_initializer([3, 3, 1, 1], np.float32, 'X3_weight') - X3_bias = generate_input_initializer([3], np.float32, 'X3_bias') - X5_weight = generate_input_initializer([3, 3, 1, 1], np.float32, 'X5_weight') - X5_bias = generate_input_initializer([3], np.float32, 'X5_bias') - - relu_node_1 = onnx.helper.make_node('Relu', ['input0'], ['X1'], name='Relu1') - conv_node_1 = onnx.helper.make_node('Conv', ['X1', 'X1_weight', 'X1_bias'], ['X2'], name='Conv1') + def construct_test_compute_range_model(self, test_model_path): + # (input) + # | + # Relu + # / \ + # Conv \ + # | \ + # Relu Conv + # | | + # Conv | + # \ / + # Add + # | + # (X6) + input = helper.make_tensor_value_info('input', TensorProto.FLOAT, [1, 3, 1, 3]) + X1_output = helper.make_tensor_value_info('X1', TensorProto.FLOAT, [1, 3, 1, 3]) + X2_output = helper.make_tensor_value_info('X2', TensorProto.FLOAT, [1, 3, 1, 3]) + X3_output = helper.make_tensor_value_info('X3', TensorProto.FLOAT, [1, 3, 1, 3]) + X4_output = helper.make_tensor_value_info('X4', TensorProto.FLOAT, [1, 3, 1, 3]) + X5_output = helper.make_tensor_value_info('X5', TensorProto.FLOAT, [1, 3, 1, 3]) + X6_output = helper.make_tensor_value_info('X6', TensorProto.FLOAT, [1, 3, 1, 3]) + W1 = generate_input_initializer([3, 3, 1, 1], np.float32, 'W1') + B1 = generate_input_initializer([3], np.float32, 'B1') + W3 = generate_input_initializer([3, 3, 1, 1], np.float32, 'W3') + B3 = generate_input_initializer([3], np.float32, 'B3') + W5 = generate_input_initializer([3, 3, 1, 1], np.float32, 'W5') + B5 = generate_input_initializer([3], np.float32, 'B5') + relu_node_1 = onnx.helper.make_node('Relu', ['input'], ['X1'], name='Relu1') + conv_node_1 = onnx.helper.make_node('Conv', ['X1', 'W1', 'B1'], ['X2'], name='Conv1') relu_node_2 = onnx.helper.make_node('Relu', ['X2'], ['X3'], name='Relu2') - conv_node_2 = onnx.helper.make_node('Conv', ['X3', 'X3_weight', 'X3_bias'], ['X4'], name='Conv2') - conv_node_3 = onnx.helper.make_node('Conv', ['X1', 'X5_weight', 'X5_bias'], ['X5'], name='Conv3') - add_node = onnx.helper.make_node('Add', ['X4', 'X5'], ['output'], name='Add') - + conv_node_2 = onnx.helper.make_node('Conv', ['X3', 'W3', 'B3'], ['X4'], name='Conv2') + conv_node_3 = onnx.helper.make_node('Conv', ['X1', 'W5', 'B5'], ['X5'], name='Conv3') + add_node = onnx.helper.make_node('Add', ['X4', 'X5'], ['X6'], name='Add') graph = helper.make_graph([relu_node_1, conv_node_1, relu_node_2, conv_node_2, conv_node_3, add_node], - 'test_graph_4', [input0], [output]) - graph.initializer.add().CopyFrom(X1_weight) - graph.initializer.add().CopyFrom(X1_bias) - graph.initializer.add().CopyFrom(X3_weight) - graph.initializer.add().CopyFrom(X3_bias) - graph.initializer.add().CopyFrom(X5_weight) - graph.initializer.add().CopyFrom(X5_bias) - + 'test_graph_4', [input], [X1_output, X2_output, X3_output, X4_output, X5_output, X6_output]) + graph.initializer.add().CopyFrom(W1) + graph.initializer.add().CopyFrom(B1) + graph.initializer.add().CopyFrom(W3) + graph.initializer.add().CopyFrom(B3) + graph.initializer.add().CopyFrom(W5) + graph.initializer.add().CopyFrom(B5) model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) - test_model_path = './test_model_4.onnx' onnx.save(model, test_model_path) - data_reader = TestDataReaderSecond() + + def test_compute_range(self): + test_model_path = './test_model_4.onnx' + self.construct_test_compute_range_model(test_model_path) + augmented_model_path = './augmented_test_model_4.onnx' - calibrater = ONNXCalibrater(test_model_path, data_reader, ['Conv', 'MatMul'], [], [], augmented_model_path) - augmented_model = calibrater.augment_graph() - onnx.save(augmented_model, augmented_model_path) + calibrater = MinMaxCalibrater(test_model_path, augmented_model_path=augmented_model_path) + data_reader = TestDataReader() + calibrater.collect_data(data_reader) + tensors_range = calibrater.compute_range() - #test calculation of quantization params - #TO_DO: check rmin/rmax - dict_for_quantization = calibrater.get_intermediate_outputs() - quantization_params_dict = calibrater.calculate_quantization_params(dict_for_quantization) + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL + infer_session = onnxruntime.InferenceSession(test_model_path, + sess_options=sess_options, + providers=['CPUExecutionProvider']) + data_reader.rewind() + rmin = np.array([np.inf, np.inf, np.inf, np.inf, np.inf, np.inf], dtype=np.float32) + rmax = -1.0 * rmin + while True: + input = data_reader.get_next() + if not input: + break + output = np.asarray(infer_session.run(None, input)).reshape(6, -1) + rmin=np.minimum(rmin, np.amin(output, axis=1)) + rmax=np.maximum(rmax, np.amax(output, axis=1)) - #check the size of the quantization dictionary - self.assertEqual(len(quantization_params_dict), 5) - - #check the computation of zp and scale - for key, value in quantization_params_dict.items(): - - self.assertTrue(value is not None) - self.assertTrue(len(value) == 2) - - thresholds = dict_for_quantization[key] - rmin = min(thresholds[0], 0) - rmax = max(thresholds[1], 0) - if key == 'X2': #next_node is Relu - if rmin < 0: rmin = 0 - - scale_expected = np.float32((rmax - rmin) / 255 if rmin != rmax else 1) - zp_expected = np.uint8(round(max(0, min(255, (0 - rmin) / scale_expected)))) - zp_actual = value[0] - scale_actual = value[1] - - self.assertEqual(zp_expected, zp_actual) - self.assertEqual(scale_expected, scale_actual) - - print('Finished' + ' test calculation of quantization params.') + min_max_pairs = list(zip(rmin, rmax)) + output_names = [infer_session.get_outputs()[i].name for i in range(len(infer_session.get_outputs()))] + output_min_max_dict = dict(zip(output_names, min_max_pairs)) + for output_name in output_min_max_dict.keys(): + self.assertEqual(output_min_max_dict[output_name], tensors_range[output_name]) if __name__ == '__main__': diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc index 402a85ac59..dd04537928 100644 --- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc +++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc @@ -39,7 +39,7 @@ #include "core/optimizer/unsqueeze_elimination.h" #include "core/session/inference_session.h" #include "orttraining/core/framework/distributed_run_context.h" -#include "orttraining/core/optimizer/bias_dropout_fusion.h" +#include "core/optimizer/bias_dropout_fusion.h" #include "orttraining/core/optimizer/concat_replacement.h" #include "orttraining/core/optimizer/insert_output_rewriter.h" #include "orttraining/core/optimizer/localized_recompute.h" diff --git a/orttraining/orttraining/core/session/training_session.cc b/orttraining/orttraining/core/session/training_session.cc index 1b0d62f977..9fe5c1bd72 100644 --- a/orttraining/orttraining/core/session/training_session.cc +++ b/orttraining/orttraining/core/session/training_session.cc @@ -497,19 +497,7 @@ Status TrainingSession::ConfigureForTraining( } } - #if 1 - // TODO: Do not merge this on master - // Saving training model before optimizer nodes are added - // This makes easier to manually edit MNIST model later - if ((IsRootNode(config) || (config.pipeline_config.has_value() && - DistributedRunContext::GroupId(WorkerGroupType::ModelParallel) == 0)) && - config.model_with_training_graph_path.has_value()) { - ORT_IGNORE_RETURN_VALUE(Save( - config.model_with_training_graph_path.value(), SaveOption::NO_RELOAD)); - } - #endif - - // add optimizer or gradient accumulation + // Add optimizer or gradient accumulation if (config.optimizer_config.has_value()) { OptimizerGraphConfig opt_graph_config{}; std::unordered_map opt_node_configs{}; @@ -574,16 +562,12 @@ Status TrainingSession::ConfigureForTraining( // conflict. It is user's responsibility to make sure different rank is passed in with different. Also, to avoid // writing conflict, only the ranks in first pipeline group write the partition file out. // model_with_training_graph_path value. - #if 0 - // TODO: Do not merge this on master - // This is being called above, before optimizers nodes are added if ((IsRootNode(config) || (config.pipeline_config.has_value() && DistributedRunContext::GroupId(WorkerGroupType::PipelineParallel) == 0)) && config.model_with_training_graph_path.has_value()) { ORT_IGNORE_RETURN_VALUE(Save( config.model_with_training_graph_path.value(), SaveOption::NO_RELOAD)); } - #endif // After pipeline partition, we need to return the inputs allowed in this partition. if (config.pipeline_config.has_value()) { diff --git a/orttraining/orttraining/python/orttraining_pybind_state.cc b/orttraining/orttraining/python/orttraining_pybind_state.cc index a45ac0825a..93c2f67e72 100644 --- a/orttraining/orttraining/python/orttraining_pybind_state.cc +++ b/orttraining/orttraining/python/orttraining_pybind_state.cc @@ -10,7 +10,7 @@ #include "core/session/environment.h" #include "orttraining/core/session/training_session.h" #include "orttraining/core/graph/optimizer_config.h" -#include "orttraining/core/framework/mpi_context.h" +#include "orttraining/core/framework/communication/mpi/mpi_context.h" #include "orttraining/core/framework/module_gradient_graph_builder.h" #include "python/onnxruntime_pybind_mlvalue.h" @@ -22,9 +22,6 @@ using namespace onnxruntime::logging; using namespace onnxruntime::training; struct TrainingParameters { - std::string model_with_loss_function_path; - std::string model_with_training_graph_path; - std::string loss_output_name; std::unordered_set weights_to_train; std::unordered_set weights_not_to_train; @@ -100,8 +97,6 @@ TrainingConfigurationResult ConfigureSessionForTraining( } training::TrainingSession::TrainingConfiguration config{}; - config.model_with_loss_function_path = parameters.model_with_loss_function_path; - config.model_with_training_graph_path = parameters.model_with_training_graph_path; config.weight_names_to_train = parameters.weights_to_train; config.weight_names_to_not_train = parameters.weights_not_to_train; config.immutable_weights = parameters.immutable_weights; @@ -296,8 +291,6 @@ std::unordered_map> Con void addObjectMethodsForTraining(py::module& m) { py::class_ parameters(m, "TrainingParameters", R"pbdoc(Configuration information for training.)pbdoc"); parameters.def(py::init()) - .def_readwrite("model_with_loss_function_path", &TrainingParameters::model_with_loss_function_path) - .def_readwrite("model_with_training_graph_path", &TrainingParameters::model_with_training_graph_path) .def_readwrite("loss_output_name", &TrainingParameters::loss_output_name) .def_readwrite("immutable_weights", &TrainingParameters::immutable_weights) .def_readwrite("weights_not_to_train", &TrainingParameters::weights_not_to_train) diff --git a/orttraining/orttraining/test/optimizer/graph_transform_test.cc b/orttraining/orttraining/test/optimizer/graph_transform_test.cc index 16ab6fc73f..bd9d99272c 100644 --- a/orttraining/orttraining/test/optimizer/graph_transform_test.cc +++ b/orttraining/orttraining/test/optimizer/graph_transform_test.cc @@ -13,7 +13,7 @@ #include "core/optimizer/bias_gelu_fusion.h" #include "core/optimizer/gelu_fusion.h" #include "core/optimizer/dropout_elimination.h" -#include "orttraining/core/optimizer/bias_dropout_fusion.h" +#include "core/optimizer/bias_dropout_fusion.h" #include "orttraining/core/optimizer/gist_encode_decode.h" #include "orttraining/core/optimizer/nonzero_shape_setter.h" #include "orttraining/core/optimizer/megatron_transformer.h" diff --git a/winml/api/Microsoft.AI.MachineLearning.Experimental.idl b/winml/api/Microsoft.AI.MachineLearning.Experimental.idl index 786a5c056d..f9806db92b 100644 --- a/winml/api/Microsoft.AI.MachineLearning.Experimental.idl +++ b/winml/api/Microsoft.AI.MachineLearning.Experimental.idl @@ -20,7 +20,6 @@ import "Windows.AI.MachineLearning.idl"; #define ROOT_NS Microsoft #endif - namespace ROOT_NS.AI.MachineLearning.Experimental { runtimeclass LearningModelBuilder; diff --git a/winml/dll/module.cpp b/winml/dll/module.cpp index 0bf99a1361..e38dcc3cf3 100644 --- a/winml/dll/module.cpp +++ b/winml/dll/module.cpp @@ -133,4 +133,4 @@ STDAPI DllGetActivationFactory(HSTRING classId, void** factory) { #endif return ret; -} \ No newline at end of file +}