diff --git a/onnxruntime/__init__.py b/onnxruntime/__init__.py index 359c07d8fc..2060c4415c 100644 --- a/onnxruntime/__init__.py +++ b/onnxruntime/__init__.py @@ -9,7 +9,14 @@ For more information on ONNX Runtime, please see `aka.ms/onnxruntime (OrtDevice::DeviceId)>; struct DeviceAllocatorRegistrationInfo { + DeviceAllocatorRegistrationInfo(OrtMemType ort_mem_type, + DeviceAllocatorFactory alloc_factory, + size_t mem, + ArenaExtendStrategy strategy = ArenaExtendStrategy::kNextPowerOfTwo) : mem_type(ort_mem_type), + factory(alloc_factory), + max_mem(mem), + arena_extend_strategy(strategy) {} OrtMemType mem_type; DeviceAllocatorFactory factory; size_t max_mem; - ArenaExtendStrategy arena_extend_strategy = ArenaExtendStrategy::kNextPowerOfTwo; + ArenaExtendStrategy arena_extend_strategy; }; AllocatorPtr CreateAllocator(DeviceAllocatorRegistrationInfo info, OrtDevice::DeviceId device_id = 0); diff --git a/orttraining/orttraining/core/framework/data_transfer_utils.h b/onnxruntime/core/framework/data_transfer_utils.h similarity index 94% rename from orttraining/orttraining/core/framework/data_transfer_utils.h rename to onnxruntime/core/framework/data_transfer_utils.h index 9df70ac838..4f08481d61 100644 --- a/orttraining/orttraining/core/framework/data_transfer_utils.h +++ b/onnxruntime/core/framework/data_transfer_utils.h @@ -47,7 +47,10 @@ common::Status CopyTensorDataToSpan( const DataTransferManager& data_transfer_manager, const Tensor& src_tensor, const OrtMemoryInfo& dst_alloc_info, gsl::span dst_span) { +// std::is_trivially_copyable is not implemented in older versions of GCC +#if !defined(__GNUC__) || __GNUC__ >= 5 static_assert(std::is_trivially_copyable::value, "Element type must be trivially copyable."); +#endif ORT_RETURN_IF_NOT(src_tensor.DataType() == DataTypeImpl::GetType()); ORT_RETURN_IF_NOT( src_tensor.SizeInBytes() == static_cast(dst_span.size_bytes())); diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index 524c528927..eb78ff89f0 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -6,6 +6,7 @@ #include #include "core/common/logging/logging.h" +#include "core/common/safeint.h" #include "core/framework/node_index_info.h" #include "core/framework/op_kernel.h" #include "core/framework/utils.h" @@ -242,12 +243,12 @@ Status SessionState::GeneratePatternGroupCache(const std::vectorShape()) continue; size_t size = 0; - int64_t len = 1; + SafeInt len = 1; for (auto& dim : arg->Shape()->dim()) { if (dim.has_dim_param()) { auto it = map.find(dim.dim_param()); if (it == map.end()) { - return Status(ONNXRUNTIME, FAIL, "Unknow shape found in memory pattern compute"); + return Status(ONNXRUNTIME, FAIL, "Unknown shape found in memory pattern compute"); } len *= it->second; } else { diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 4d656dd4a3..5a0d9b32db 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +// needs to be included first to get around onnxruntime\cmake\external\onnx\onnx/common/constants.h(14): error C2513: 'bool': no variable declared before '=' +#include "core/framework/tensorprotoutils.h" + #include "onnxruntime_pybind_mlvalue.h" #define NO_IMPORT_ARRAY diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e311c899a7..ceedb14822 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1,22 +1,25 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "onnxruntime_pybind_exceptions.h" -#include "onnxruntime_pybind_mlvalue.h" +// needs to be included first to get around onnxruntime\cmake\external\onnx\onnx/common/constants.h(14): error C2513: 'bool': no variable declared before '=' +#include "core/framework/tensorprotoutils.h" + +#include "python/onnxruntime_pybind_exceptions.h" +#include "python/onnxruntime_pybind_mlvalue.h" #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #define PY_ARRAY_UNIQUE_SYMBOL onnxruntime_python_ARRAY_API #include +#include "core/framework/data_transfer_utils.h" #include "core/framework/data_types_internal.h" -#include "core/framework/tensorprotoutils.h" #include "core/graph/graph_viewer.h" #include "core/common/logging/logging.h" #include "core/common/logging/severity.h" #include "core/framework/TensorSeq.h" #include "core/framework/session_options.h" #include "core/framework/bfc_arena.h" -#include "core/framework/random_seed.h" +#include "core/session/IOBinding.h" #if USE_CUDA #define BACKEND_PROC "GPU" @@ -88,16 +91,15 @@ #include "core/providers/cpu/cpu_provider_factory.h" #ifdef ENABLE_TRAINING -#include "core/session/IOBinding.h" +#include "core/framework/random_seed.h" #include "orttraining/core/session/training_session.h" #include "orttraining/core/graph/optimizer_config.h" #include "orttraining/core/framework/mpi_setup.h" -#include "orttraining/core/framework/data_transfer_utils.h" -#endif +#endif // ENABLE_TRAINING #ifdef USE_CUDA #include "core/providers/cuda/cuda_provider_factory.h" -int cuda_device_id = 0; +OrtDevice::DeviceId cuda_device_id = 0; size_t cuda_mem_limit = std::numeric_limits::max(); onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; #endif @@ -141,7 +143,6 @@ std::shared_ptr CreateExecutionProviderFactory_Nuphar #endif // _MSC_VER namespace onnxruntime { -using namespace training; namespace python { namespace py = pybind11; @@ -215,34 +216,6 @@ static std::string GetDeviceName(const OrtDevice& device) { } } -struct TrainingParameters { - std::string loss_output_name; - std::unordered_set weights_to_train; - std::unordered_set weights_not_to_train; - onnxruntime::training::TrainingSession::ImmutableWeights immutable_weights; - - // optimizer - std::string training_optimizer_name; - std::string loss_scale_input_name; - std::string scaled_loss_output_name; - std::string lr_params_feed_name = "Learning_Rate"; - std::unordered_map> optimizer_attributes_map; - bool use_fp16_moments = false; - - bool use_mixed_precision = false; - bool allreduce_post_accumulation = false; - float loss_scale = 0.0f; - int world_rank = 0; - int world_size = 1; - int local_rank = 0; - int local_size = 1; - int gradient_accumulation_steps = 1; - int data_parallel_size = 1; - int horizontal_parallel_size = 1; - bool partition_optimizer = false; - int seed = -1; -}; - template <> void AddNonTensor(OrtValue& val, std::vector& pyobjs) { const auto& seq_tensors = val.Get(); @@ -409,99 +382,6 @@ void InitializeSession(InferenceSession* sess, const std::vector& p OrtPybindThrowIfError(sess->Initialize()); } -// TODO: this method does not handle parallel optimization. -static void ConfigureSessionForTraining( - training::TrainingSession* sess, TrainingParameters& parameters) { - //TODO tix, refactor the mpi related code to populate all fields correctly by default. - ORT_ENFORCE(parameters.horizontal_parallel_size <= parameters.world_size); - ORT_ENFORCE(parameters.data_parallel_size <= parameters.world_size); - if (parameters.world_size % parameters.horizontal_parallel_size != 0) { - throw std::runtime_error("Cannot split horizontal parallel group because world_size is not divisible"); - } - - auto data_group_size = parameters.world_size / parameters.horizontal_parallel_size; - if (data_group_size != parameters.data_parallel_size) { - std::cout << "WARNING: data_parallel_size is not correct, tuned automatically to " - << data_group_size << std::endl; - parameters.data_parallel_size = data_group_size; - } -#ifdef USE_HOROVOD - // this condition block is temporary. - // For now, nccl allreduce kernel only implements for allreduce_post_accumulation - // hovorod allreduce kernel only implements for not allreduce_post_accumulation. - bool use_nccl = parameters.allreduce_post_accumulation; - if (!use_nccl && parameters.world_size > 1) { - auto mpi_context = setup_horovod(); - std::cout << "mpi_context.world_rank: " << mpi_context.world_rank << std::endl; - std::cout << "mpi_context.local_rank: " << mpi_context.local_rank << std::endl; - std::cout << "mpi_context.world_size: " << mpi_context.world_size << std::endl; - std::cout << "mpi_context.local_size: " << mpi_context.local_size << std::endl; - parameters.local_size = mpi_context.local_size; - parameters.local_rank = mpi_context.local_rank; - } -#endif - - training::TrainingSession::TrainingConfiguration config{}; - config.weight_names_to_train = parameters.weights_to_train; - config.weight_names_to_not_train = parameters.weights_not_to_train; - config.immutable_weights = parameters.immutable_weights; - - config.set_gradients_as_graph_outputs = true; - - config.gradient_accumulation_steps = parameters.gradient_accumulation_steps; - - config.distributed_config.world_rank = parameters.world_rank; - config.distributed_config.world_size = parameters.world_size; - config.distributed_config.local_rank = parameters.local_rank; - config.distributed_config.local_size = parameters.local_size; - config.distributed_config.data_parallel_size = parameters.data_parallel_size; - config.distributed_config.horizontal_parallel_size = parameters.horizontal_parallel_size; - - if (parameters.use_mixed_precision) { - training::TrainingSession::TrainingConfiguration::MixedPrecisionConfiguration mp{}; - mp.add_loss_scaling = false; - mp.use_fp16_initializers = true; - - config.mixed_precision_config = mp; - } - - config.loss_name = - parameters.use_mixed_precision ? parameters.scaled_loss_output_name : parameters.loss_output_name; - - if (!parameters.training_optimizer_name.empty()) { - training::TrainingSession::TrainingConfiguration::OptimizerConfiguration opt{}; - opt.name = parameters.training_optimizer_name; - opt.learning_rate_input_name = parameters.lr_params_feed_name; - opt.weight_attributes_generator = [¶meters](const std::string& weight_name) { - const auto it = parameters.optimizer_attributes_map.find(weight_name); - ORT_ENFORCE( - it != parameters.optimizer_attributes_map.end(), - "Failed to find attribute map for weight ", weight_name); - return it->second; - }; - opt.use_fp16_moments = parameters.use_fp16_moments; - opt.do_all_reduce_in_fp16 = true; - // TODO: this mapping is temporary. - // For now, nccl allreduce kernel only implements for allreduce_post_accumulation - // hovorod allreduce kernel only implements for not allreduce_post_accumulation. - // eventually we will have one all reduce kernel and let opt to have - // an allreduce_post_accumulation option and remove the use_nccl option. - opt.use_nccl = parameters.allreduce_post_accumulation; - opt.partition_optimizer = parameters.partition_optimizer; - - config.optimizer_config = opt; - } - - if (parameters.seed > 0) { - utils::SetStaticRandomSeed(static_cast(parameters.seed)); - std::cout << "Random seed is set to " << parameters.seed << std::endl; - } - - training::TrainingSession::TrainingConfigurationResult config_result{}; - - OrtPybindThrowIfError(sess->ConfigureForTraining(config, config_result)); -} - void addGlobalMethods(py::module& m) { m.def("get_default_session_options", &GetDefaultCPUSessionOptions, "Return a default session_options instance."); m.def("get_session_initializer", &SessionObjectInitializer::Get, "Return a default session object initializer."); @@ -586,7 +466,7 @@ void addGlobalMethods(py::module& m) { #endif //onnxruntime_PYBIND_EXPORT_OPSCHEMA #ifdef USE_CUDA - m.def("set_cuda_device_id", [](const int id) { cuda_device_id = id; }); + m.def("set_cuda_device_id", [](const int id) { cuda_device_id = static_cast(id); }); m.def("set_cuda_mem_limit", [](const int64_t limit) { cuda_mem_limit = static_cast(limit); }); @@ -775,26 +655,6 @@ void addObjectMethods(py::module& m) { io_binding->Get()->ClearOutputs(); }); - py::class_ parameters(m, "TrainingParameters", R"pbdoc(Configuration information for training.)pbdoc"); - parameters.def(py::init()) - .def_readwrite("loss_output_name", &TrainingParameters::loss_output_name) - .def_readwrite("immutable_weights", &TrainingParameters::immutable_weights) - .def_readwrite("weights_not_to_train", &TrainingParameters::weights_not_to_train) - .def_readwrite("weights_to_train", &TrainingParameters::weights_to_train) - .def_readwrite("loss_scale_input_name", &TrainingParameters::loss_scale_input_name) - .def_readwrite("scaled_loss_output_name", &TrainingParameters::scaled_loss_output_name) - .def_readwrite("training_optimizer_name", &TrainingParameters::training_optimizer_name) - .def_readwrite("lr_params_feed_name", &TrainingParameters::lr_params_feed_name) - .def_readwrite("optimizer_attributes_map", &TrainingParameters::optimizer_attributes_map) - .def_readwrite("use_fp16_moments", &TrainingParameters::use_fp16_moments) - .def_readwrite("use_mixed_precision", &TrainingParameters::use_mixed_precision) - .def_readwrite("allreduce_post_accumulation", &TrainingParameters::allreduce_post_accumulation) - .def_readwrite("loss_scale", &TrainingParameters::loss_scale) - .def_readwrite("world_rank", &TrainingParameters::world_rank) - .def_readwrite("world_size", &TrainingParameters::world_size) - .def_readwrite("gradient_accumulation_steps", &TrainingParameters::gradient_accumulation_steps) - .def_readwrite("partition_optimizer", &TrainingParameters::partition_optimizer); - py::class_ sess(m, "SessionOptions", R"pbdoc(Configuration information for a session.)pbdoc"); sess @@ -1057,12 +917,170 @@ including arg name, arg type (contains both type and shape).)pbdoc") throw std::runtime_error("Error in execution: " + status.ErrorMessage()); }); + py::enum_(m, "ArenaExtendStrategy", py::arithmetic()) + .value("kNextPowerOfTwo", onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo) + .value("kSameAsRequested", onnxruntime::ArenaExtendStrategy::kSameAsRequested) + .export_values(); +} + +#if defined(USE_MIMALLOC_ARENA_ALLOCATOR) +static struct { + PyMemAllocatorEx mem; + PyMemAllocatorEx raw; + PyMemAllocatorEx obj; +} allocators; +#endif + +#ifdef ENABLE_TRAINING +namespace { +struct TrainingParameters { + std::string loss_output_name; + std::unordered_set weights_to_train; + std::unordered_set weights_not_to_train; + onnxruntime::training::TrainingSession::ImmutableWeights immutable_weights; + + // optimizer + std::string training_optimizer_name; + std::string loss_scale_input_name; + std::string scaled_loss_output_name; + std::string lr_params_feed_name = "Learning_Rate"; + std::unordered_map> optimizer_attributes_map; + bool use_fp16_moments = false; + + bool use_mixed_precision = false; + bool allreduce_post_accumulation = false; + float loss_scale = 0.0f; + int world_rank = 0; + int world_size = 1; + int local_rank = 0; + int local_size = 1; + int gradient_accumulation_steps = 1; + int data_parallel_size = 1; + int horizontal_parallel_size = 1; + bool partition_optimizer = false; + int seed = -1; +}; + +// TODO: this method does not handle parallel optimization. +void ConfigureSessionForTraining( + training::TrainingSession* sess, TrainingParameters& parameters) { + //TODO tix, refactor the mpi related code to populate all fields correctly by default. + ORT_ENFORCE(parameters.horizontal_parallel_size <= parameters.world_size); + ORT_ENFORCE(parameters.data_parallel_size <= parameters.world_size); + if (parameters.world_size % parameters.horizontal_parallel_size != 0) { + throw std::runtime_error("Cannot split horizontal parallel group because world_size is not divisible"); + } + + auto data_group_size = parameters.world_size / parameters.horizontal_parallel_size; + if (data_group_size != parameters.data_parallel_size) { + std::cout << "WARNING: data_parallel_size is not correct, tuned automatically to " + << data_group_size << std::endl; + parameters.data_parallel_size = data_group_size; + } +#ifdef USE_HOROVOD + // this condition block is temporary. + // For now, nccl allreduce kernel only implements for allreduce_post_accumulation + // hovorod allreduce kernel only implements for not allreduce_post_accumulation. + bool use_nccl = parameters.allreduce_post_accumulation; + if (!use_nccl && parameters.world_size > 1) { + auto mpi_context = training::setup_horovod(); + std::cout << "mpi_context.world_rank: " << mpi_context.world_rank << std::endl; + std::cout << "mpi_context.local_rank: " << mpi_context.local_rank << std::endl; + std::cout << "mpi_context.world_size: " << mpi_context.world_size << std::endl; + std::cout << "mpi_context.local_size: " << mpi_context.local_size << std::endl; + parameters.local_size = mpi_context.local_size; + parameters.local_rank = mpi_context.local_rank; + } +#endif + + training::TrainingSession::TrainingConfiguration config{}; + config.weight_names_to_train = parameters.weights_to_train; + config.weight_names_to_not_train = parameters.weights_not_to_train; + config.immutable_weights = parameters.immutable_weights; + + config.set_gradients_as_graph_outputs = true; + + config.gradient_accumulation_steps = parameters.gradient_accumulation_steps; + + config.distributed_config.world_rank = parameters.world_rank; + config.distributed_config.world_size = parameters.world_size; + config.distributed_config.local_rank = parameters.local_rank; + config.distributed_config.local_size = parameters.local_size; + config.distributed_config.data_parallel_size = parameters.data_parallel_size; + config.distributed_config.horizontal_parallel_size = parameters.horizontal_parallel_size; + + if (parameters.use_mixed_precision) { + training::TrainingSession::TrainingConfiguration::MixedPrecisionConfiguration mp{}; + mp.add_loss_scaling = false; + mp.use_fp16_initializers = true; + + config.mixed_precision_config = mp; + } + + config.loss_name = + parameters.use_mixed_precision ? parameters.scaled_loss_output_name : parameters.loss_output_name; + + if (!parameters.training_optimizer_name.empty()) { + training::TrainingSession::TrainingConfiguration::OptimizerConfiguration opt{}; + opt.name = parameters.training_optimizer_name; + opt.learning_rate_input_name = parameters.lr_params_feed_name; + opt.weight_attributes_generator = [¶meters](const std::string& weight_name) { + const auto it = parameters.optimizer_attributes_map.find(weight_name); + ORT_ENFORCE( + it != parameters.optimizer_attributes_map.end(), + "Failed to find attribute map for weight ", weight_name); + return it->second; + }; + opt.use_fp16_moments = parameters.use_fp16_moments; + opt.do_all_reduce_in_fp16 = true; + // TODO: this mapping is temporary. + // For now, nccl allreduce kernel only implements for allreduce_post_accumulation + // hovorod allreduce kernel only implements for not allreduce_post_accumulation. + // eventually we will have one all reduce kernel and let opt to have + // an allreduce_post_accumulation option and remove the use_nccl option. + opt.use_nccl = parameters.allreduce_post_accumulation; + opt.partition_optimizer = parameters.partition_optimizer; + + config.optimizer_config = opt; + } + + if (parameters.seed > 0) { + utils::SetStaticRandomSeed(static_cast(parameters.seed)); + std::cout << "Random seed is set to " << parameters.seed << std::endl; + } + + training::TrainingSession::TrainingConfigurationResult config_result{}; + + OrtPybindThrowIfError(sess->ConfigureForTraining(config, config_result)); +} + +void addObjectMethodsForTraining(py::module& m) { + py::class_ parameters(m, "TrainingParameters", R"pbdoc(Configuration information for training.)pbdoc"); + parameters.def(py::init()) + .def_readwrite("loss_output_name", &TrainingParameters::loss_output_name) + .def_readwrite("immutable_weights", &TrainingParameters::immutable_weights) + .def_readwrite("weights_not_to_train", &TrainingParameters::weights_not_to_train) + .def_readwrite("weights_to_train", &TrainingParameters::weights_to_train) + .def_readwrite("loss_scale_input_name", &TrainingParameters::loss_scale_input_name) + .def_readwrite("scaled_loss_output_name", &TrainingParameters::scaled_loss_output_name) + .def_readwrite("training_optimizer_name", &TrainingParameters::training_optimizer_name) + .def_readwrite("lr_params_feed_name", &TrainingParameters::lr_params_feed_name) + .def_readwrite("optimizer_attributes_map", &TrainingParameters::optimizer_attributes_map) + .def_readwrite("use_fp16_moments", &TrainingParameters::use_fp16_moments) + .def_readwrite("use_mixed_precision", &TrainingParameters::use_mixed_precision) + .def_readwrite("allreduce_post_accumulation", &TrainingParameters::allreduce_post_accumulation) + .def_readwrite("loss_scale", &TrainingParameters::loss_scale) + .def_readwrite("world_rank", &TrainingParameters::world_rank) + .def_readwrite("world_size", &TrainingParameters::world_size) + .def_readwrite("gradient_accumulation_steps", &TrainingParameters::gradient_accumulation_steps) + .def_readwrite("partition_optimizer", &TrainingParameters::partition_optimizer); + py::class_ training_session(m, "TrainingSession"); training_session.def(py::init()) .def(py::init()) .def("finalize", [](py::object) { #ifdef USE_HOROVOD - shutdown_horovod(); + training::shutdown_horovod(); #endif }) .def("load_model", [](onnxruntime::training::TrainingSession* sess, const std::string& path, TrainingParameters& parameters) { @@ -1126,20 +1144,9 @@ including arg name, arg type (contains both type and shape).)pbdoc") } ORT_THROW_IF_ERROR(sess->SetStateTensors(state_tensors, strict)); }); - - py::enum_(m, "ArenaExtendStrategy", py::arithmetic()) - .value("kNextPowerOfTwo", onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo) - .value("kSameAsRequested", onnxruntime::ArenaExtendStrategy::kSameAsRequested) - .export_values(); } - -#if defined(USE_MIMALLOC_ARENA_ALLOCATOR) -static struct { - PyMemAllocatorEx mem; - PyMemAllocatorEx raw; - PyMemAllocatorEx obj; -} allocators; -#endif +} // namespace +#endif // ENABLE_TRAINING PYBIND11_MODULE(onnxruntime_pybind11_state, m) { m.doc() = "pybind11 stateful interface to ONNX runtime"; @@ -1210,6 +1217,10 @@ PYBIND11_MODULE(onnxruntime_pybind11_state, m) { addGlobalMethods(m); addObjectMethods(m); +#ifdef ENABLE_TRAINING + addObjectMethodsForTraining(m); +#endif // ENABLE_TRAINING + #ifdef onnxruntime_PYBIND_EXPORT_OPSCHEMA addOpSchemaSubmodule(m); addOpKernelSubmodule(m); diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 79de5b06b1..62576fe23a 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -461,7 +461,7 @@ std::unique_ptr OpTester::BuildGraph( std::vector node_input_defs; std::vector output_defs; - for (auto i = 0; i < input_data_.size(); ++i) { + for (size_t i = 0; i < input_data_.size(); ++i) { node_input_defs.push_back(&input_data_[i].def_); } diff --git a/orttraining/orttraining/core/framework/checkpointing.cc b/orttraining/orttraining/core/framework/checkpointing.cc index 527be9ffac..4652ccffcd 100644 --- a/orttraining/orttraining/core/framework/checkpointing.cc +++ b/orttraining/orttraining/core/framework/checkpointing.cc @@ -11,13 +11,13 @@ #include "core/common/common.h" #include "core/common/logging/logging.h" #include "core/common/path.h" +#include "core/framework/data_transfer_utils.h" #include "core/framework/endian_utils.h" #include "core/framework/ml_value.h" #include "core/framework/path_lib.h" #include "core/framework/tensor_external_data_info.h" #include "core/framework/tensorprotoutils.h" #include "core/platform/env.h" -#include "orttraining/core/framework/data_transfer_utils.h" #include "orttraining/core/framework/protobuf_message_sequence.h" #include "core/util/protobuf_parsing_utils.h" diff --git a/orttraining/orttraining/core/session/training_session.cc b/orttraining/orttraining/core/session/training_session.cc index 97929dde53..a39331cc58 100644 --- a/orttraining/orttraining/core/session/training_session.cc +++ b/orttraining/orttraining/core/session/training_session.cc @@ -5,11 +5,11 @@ #include "orttraining/core/session/training_session.h" +#include "core/framework/data_transfer_utils.h" #include "core/graph/model.h" #include "orttraining/core/graph/loss_function_builder.h" #include "orttraining/core/graph/optimizer_builder.h" #include "orttraining/core/framework/checkpointing.h" -#include "orttraining/core/framework/data_transfer_utils.h" #include "orttraining/core/framework/gradient_graph_builder.h" #include "orttraining/core/framework/distributed_run_context.h" #include "orttraining/core/graph/optimizer_graph_builder_registry.h"