From ec3da68247adc7d69a6584e0a518513f0c08fc3d Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Wed, 6 Apr 2022 00:35:00 +0000 Subject: [PATCH] On device training offline composition --- cmake/CMakeLists.txt | 12 + cmake/onnxruntime_on_device_training.cmake | 52 ++ orttraining/orttraining/onnxflow/README.md | 28 + .../onnxflow/csrc/load_parameters.cpp | 26 + .../onnxflow/csrc/load_parameters.h | 9 + .../orttraining/onnxflow/csrc/onnxflow.pb.cc | 543 ++++++++++++++++ .../orttraining/onnxflow/csrc/onnxflow.pb.h | 602 ++++++++++++++++++ .../onnxflow/models/simple_model.onnx | Bin 0 -> 680 bytes .../orttraining/onnxflow/onnxflow.proto | 17 + .../orttraining/onnxflow/onnxflow/__init__.py | 7 + .../orttraining/onnxflow/onnxflow/graph.py | 179 ++++++ .../orttraining/onnxflow/onnxflow/loss.py | 136 ++++ .../onnxflow/onnxflow/onnxflow_pb2.py | 129 ++++ .../orttraining/onnxflow/onnxflow/optim.py | 144 +++++ orttraining/orttraining/onnxflow/sample.m.cpp | 32 + orttraining/orttraining/onnxflow/sample.py | 36 ++ tools/ci_build/build.py | 6 + 17 files changed, 1958 insertions(+) create mode 100644 cmake/onnxruntime_on_device_training.cmake create mode 100644 orttraining/orttraining/onnxflow/README.md create mode 100644 orttraining/orttraining/onnxflow/csrc/load_parameters.cpp create mode 100644 orttraining/orttraining/onnxflow/csrc/load_parameters.h create mode 100644 orttraining/orttraining/onnxflow/csrc/onnxflow.pb.cc create mode 100644 orttraining/orttraining/onnxflow/csrc/onnxflow.pb.h create mode 100644 orttraining/orttraining/onnxflow/models/simple_model.onnx create mode 100644 orttraining/orttraining/onnxflow/onnxflow.proto create mode 100644 orttraining/orttraining/onnxflow/onnxflow/__init__.py create mode 100644 orttraining/orttraining/onnxflow/onnxflow/graph.py create mode 100644 orttraining/orttraining/onnxflow/onnxflow/loss.py create mode 100644 orttraining/orttraining/onnxflow/onnxflow/onnxflow_pb2.py create mode 100644 orttraining/orttraining/onnxflow/onnxflow/optim.py create mode 100644 orttraining/orttraining/onnxflow/sample.m.cpp create mode 100644 orttraining/orttraining/onnxflow/sample.py diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e5793057a2..0007f735a0 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -160,6 +160,9 @@ option(onnxruntime_ENABLE_BITCODE "Enable bitcode for iOS only" OFF) # build eager mode option(onnxruntime_ENABLE_EAGER_MODE "build ort eager mode") +# build on device training mode +option(onnxruntime_ENABLE_ON_DEVICE_TRAINING "build ort on device training") + # build separate library of schemas of (custom) ops used by ORT (for ONNX to MLIR translation) option(onnxruntime_BUILD_OPSCHEMA_LIB "Build op schema library" ON) @@ -1935,6 +1938,15 @@ if (onnxruntime_ENABLE_EAGER_MODE) add_compile_definitions(ENABLE_EAGER_MODE) list(APPEND ONNXRUNTIME_TARGETS onnxruntime_eager) endif() +if (onnxruntime_ENABLE_ON_DEVICE_TRAINING) + if (NOT onnxruntime_ENABLE_TRAINING) + message( + FATAL_ERROR + "Option onnxruntime_ENABLE_ON_DEVICE_TRAINING can only be used when onnxruntime_ENABLE_TRAINING is enabled") + endif() + add_compile_definitions(ENABLE_ON_DEVICE_TRAINING) + list(APPEND ONNXRUNTIME_TARGETS onnxruntime_on_device_training) +endif() foreach(target_name ${ONNXRUNTIME_TARGETS}) include(${target_name}.cmake) endforeach() diff --git a/cmake/onnxruntime_on_device_training.cmake b/cmake/onnxruntime_on_device_training.cmake new file mode 100644 index 0000000000..9aed1d6c24 --- /dev/null +++ b/cmake/onnxruntime_on_device_training.cmake @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +set(_onnxflow_pb_cpp_srcs + "${ORTTRAINING_ROOT}/orttraining/onnxflow/csrc/onnxflow.pb.cc" + "${ORTTRAINING_ROOT}/orttraining/onnxflow/csrc/onnxflow.pb.h" + ) + +if(EXISTS "${ONNX_CUSTOM_PROTOC_EXECUTABLE}") + set(PROTOC_EXECUTABLE ${ONNX_CUSTOM_PROTOC_EXECUTABLE}) +else() + set(PROTOC_EXECUTABLE $) + set(PROTOC_DEPS protobuf::protoc) +endif() + +add_custom_command( + OUTPUT ${_onnxflow_pb_cpp_srcs} + COMMAND ${PROTOC_EXECUTABLE} + ARGS --python_out=${ORTTRAINING_ROOT}/orttraining/onnxflow/onnxflow/ --cpp_out=${ORTTRAINING_ROOT}/orttraining/onnxflow/csrc/ --proto_path=${ORTTRAINING_ROOT}/orttraining/onnxflow --proto_path=${REPO_ROOT}/cmake/external/protobuf/src ${ORTTRAINING_ROOT}/orttraining/onnxflow/onnxflow.proto + DEPENDS ${ORTTRAINING_ROOT}/orttraining/onnxflow/onnxflow.proto ${PROTOC_DEPS} + COMMENT "Running cpp protocol buffer compiler on onnxflow.proto" + VERBATIM ) + +file(GLOB onnxruntime_on_device_training_srcs CONFIGURE_DEPENDS + "${ORTTRAINING_ROOT}/orttraining/onnxflow/csrc/*.h" + "${ORTTRAINING_ROOT}/orttraining/onnxflow/csrc/*.cpp" + ) +list(APPEND onnxruntime_on_device_training_srcs ${_onnxflow_pb_cpp_srcs}) + +source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_on_device_training_srcs}) + +onnxruntime_add_static_library(onnxruntime_on_device_training ${onnxruntime_on_device_training_srcs}) + +onnxruntime_add_include_to_target(onnxruntime_on_device_training onnxruntime_common onnxruntime_framework onnxruntime_optimizer onnxruntime_graph onnx onnx_proto ${PROTOBUF_LIB} flatbuffers) +target_include_directories(onnxruntime_on_device_training PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +add_dependencies(onnxruntime_on_device_training ${onnxruntime_EXTERNAL_DEPENDENCIES}) +set_target_properties(onnxruntime_on_device_training PROPERTIES FOLDER "ONNXRuntime") +if (onnxruntime_ENABLE_TRAINING) + target_include_directories(onnxruntime_session PRIVATE ${ORTTRAINING_ROOT}) +endif() + +# sample loading of file +file(GLOB orttraining_on_device_sample_src CONFIGURE_DEPENDS + "${ORTTRAINING_ROOT}/orttraining/onnxflow/sample.m.cpp" + ) +onnxruntime_add_executable(orttraining_on_device_sample ${orttraining_on_device_sample_src}) +onnxruntime_add_include_to_target(orttraining_on_device_sample onnxruntime_on_device_training onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} onnxruntime_training flatbuffers) +target_include_directories(orttraining_on_device_sample PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${eigen_INCLUDE_DIRS} ${CXXOPTS} ${extra_includes} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/onnx onnxruntime_training_runner ${PROTOBUF_LIB}) + +target_link_libraries(orttraining_on_device_sample PRIVATE onnxruntime_on_device_training onnx onnx_proto onnxruntime_training ${ONNXRUNTIME_LIBS} ${onnxruntime_EXTERNAL_LIBRARIES} libprotobuf) +# set_target_properties(onnxruntime_training_mnist PROPERTIES FOLDER "ONNXRuntimeTest") + diff --git a/orttraining/orttraining/onnxflow/README.md b/orttraining/orttraining/onnxflow/README.md new file mode 100644 index 0000000000..04f96a7b18 --- /dev/null +++ b/orttraining/orttraining/onnxflow/README.md @@ -0,0 +1,28 @@ +# onnxflow + + +1. Build onnxruntime with on device training flag: +```sh +./build.sh --config RelWithDebInfo --enable_training --use_cuda --cuda_home /usr/local/cuda/ --cudnn_home /usr/local/cuda/ --build_wheel --parallel --cuda_version=11.3 --skip_tests --build_wheel --build_on_device_training +``` + +This will generate the protobuf files needed for serialization and deserialization of the parameters: +- orttraining/orttraining/onnxflow/csrc/onnxflow.pb.h +- orttraining/orttraining/onnxflow/csrc/onnxflow.pb.cc +- orttraining/orttraining/onnxflow/onnxflow/onnxflow_pb2.py + +2. Compose the model with the necessary loss and optimizer by running this from `orttraining/orttraining/onnxflow`: +```py +python sample.py +``` + +This will create the following: +- Forward+Loss+Backward training onnx graph +- Optimizer onnx graph +- Serialized parameters (saved as `parameters.of`) + +3. Use the saved onnx files and the parameters to perform training. To load the serialized parameters, see example utility `orttraining/orttraining/onnxflow/sample.m.cpp`. And run it by executing +```sh +orttraining_on_device_sample +``` +Pass in the absolute path of the `parameters.of` file when prompted. diff --git a/orttraining/orttraining/onnxflow/csrc/load_parameters.cpp b/orttraining/orttraining/onnxflow/csrc/load_parameters.cpp new file mode 100644 index 0000000000..23ebeffde0 --- /dev/null +++ b/orttraining/orttraining/onnxflow/csrc/load_parameters.cpp @@ -0,0 +1,26 @@ + +#include "load_parameters.h" +#include +#include +#include +#include +#include +#include + + +namespace onnxflow { + +OnnxFlowParameters load_parameters(const std::string& path_to_file) +{ + GOOGLE_PROTOBUF_VERIFY_VERSION; + + OnnxFlowParameters params; + std::ifstream t(path_to_file); + std::stringstream buffer; + buffer << t.rdbuf(); + params.ParseFromString(buffer.str()); + + return params; +} + +} // end namespace onnxflow diff --git a/orttraining/orttraining/onnxflow/csrc/load_parameters.h b/orttraining/orttraining/onnxflow/csrc/load_parameters.h new file mode 100644 index 0000000000..849677806f --- /dev/null +++ b/orttraining/orttraining/onnxflow/csrc/load_parameters.h @@ -0,0 +1,9 @@ + + +#include "onnxflow.pb.h" + +namespace onnxflow { + +OnnxFlowParameters load_parameters(const std::string& path_to_file); + +} // end namespace onnxflow diff --git a/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.cc b/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.cc new file mode 100644 index 0000000000..ff989845a1 --- /dev/null +++ b/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.cc @@ -0,0 +1,543 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: onnxflow.proto + +#include "onnxflow.pb.h" + +#include + +#include +#include +#include +#include +// @@protoc_insertion_point(includes) +#include + +PROTOBUF_PRAGMA_INIT_SEG +namespace onnxflow { +constexpr OnnxFlowParameter::OnnxFlowParameter( + ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized) + : data_(nullptr) + , requires_grad_(false) + , is_parameter_(false){} +struct OnnxFlowParameterDefaultTypeInternal { + constexpr OnnxFlowParameterDefaultTypeInternal() + : _instance(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized{}) {} + ~OnnxFlowParameterDefaultTypeInternal() {} + union { + OnnxFlowParameter _instance; + }; +}; +PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT OnnxFlowParameterDefaultTypeInternal _OnnxFlowParameter_default_instance_; +constexpr OnnxFlowParameters::OnnxFlowParameters( + ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized) + : parameters_(){} +struct OnnxFlowParametersDefaultTypeInternal { + constexpr OnnxFlowParametersDefaultTypeInternal() + : _instance(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized{}) {} + ~OnnxFlowParametersDefaultTypeInternal() {} + union { + OnnxFlowParameters _instance; + }; +}; +PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT OnnxFlowParametersDefaultTypeInternal _OnnxFlowParameters_default_instance_; +} // namespace onnxflow +namespace onnxflow { + +// =================================================================== + +class OnnxFlowParameter::_Internal { + public: + using HasBits = decltype(std::declval()._has_bits_); + static const ::PROTOBUF_NAMESPACE_ID::Any& data(const OnnxFlowParameter* msg); + static void set_has_data(HasBits* has_bits) { + (*has_bits)[0] |= 1u; + } + static void set_has_requires_grad(HasBits* has_bits) { + (*has_bits)[0] |= 2u; + } + static void set_has_is_parameter(HasBits* has_bits) { + (*has_bits)[0] |= 4u; + } + static bool MissingRequiredFields(const HasBits& has_bits) { + return ((has_bits[0] & 0x00000007) ^ 0x00000007) != 0; + } +}; + +const ::PROTOBUF_NAMESPACE_ID::Any& +OnnxFlowParameter::_Internal::data(const OnnxFlowParameter* msg) { + return *msg->data_; +} +void OnnxFlowParameter::clear_data() { + if (data_ != nullptr) data_->Clear(); + _has_bits_[0] &= ~0x00000001u; +} +OnnxFlowParameter::OnnxFlowParameter(::PROTOBUF_NAMESPACE_ID::Arena* arena, + bool is_message_owned) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena, is_message_owned) { + SharedCtor(); + if (!is_message_owned) { + RegisterArenaDtor(arena); + } + // @@protoc_insertion_point(arena_constructor:onnxflow.OnnxFlowParameter) +} +OnnxFlowParameter::OnnxFlowParameter(const OnnxFlowParameter& from) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), + _has_bits_(from._has_bits_) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + if (from._internal_has_data()) { + data_ = new ::PROTOBUF_NAMESPACE_ID::Any(*from.data_); + } else { + data_ = nullptr; + } + ::memcpy(&requires_grad_, &from.requires_grad_, + static_cast(reinterpret_cast(&is_parameter_) - + reinterpret_cast(&requires_grad_)) + sizeof(is_parameter_)); + // @@protoc_insertion_point(copy_constructor:onnxflow.OnnxFlowParameter) +} + +void OnnxFlowParameter::SharedCtor() { +::memset(reinterpret_cast(this) + static_cast( + reinterpret_cast(&data_) - reinterpret_cast(this)), + 0, static_cast(reinterpret_cast(&is_parameter_) - + reinterpret_cast(&data_)) + sizeof(is_parameter_)); +} + +OnnxFlowParameter::~OnnxFlowParameter() { + // @@protoc_insertion_point(destructor:onnxflow.OnnxFlowParameter) + if (GetArenaForAllocation() != nullptr) return; + SharedDtor(); + _internal_metadata_.Delete(); +} + +inline void OnnxFlowParameter::SharedDtor() { + GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); + if (this != internal_default_instance()) delete data_; +} + +void OnnxFlowParameter::ArenaDtor(void* object) { + OnnxFlowParameter* _this = reinterpret_cast< OnnxFlowParameter* >(object); + (void)_this; +} +void OnnxFlowParameter::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} +void OnnxFlowParameter::SetCachedSize(int size) const { + _cached_size_.Set(size); +} + +void OnnxFlowParameter::Clear() { +// @@protoc_insertion_point(message_clear_start:onnxflow.OnnxFlowParameter) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + cached_has_bits = _has_bits_[0]; + if (cached_has_bits & 0x00000001u) { + GOOGLE_DCHECK(data_ != nullptr); + data_->Clear(); + } + ::memset(&requires_grad_, 0, static_cast( + reinterpret_cast(&is_parameter_) - + reinterpret_cast(&requires_grad_)) + sizeof(is_parameter_)); + _has_bits_.Clear(); + _internal_metadata_.Clear(); +} + +const char* OnnxFlowParameter::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + _Internal::HasBits has_bits{}; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + switch (tag >> 3) { + // required .google.protobuf.Any data = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ctx->ParseMessage(_internal_mutable_data(), ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; + // required bool requires_grad = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + _Internal::set_has_requires_grad(&has_bits); + requires_grad_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; + // required bool is_parameter = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + _Internal::set_has_is_parameter(&has_bits); + is_parameter_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; + default: + goto handle_unusual; + } // switch + handle_unusual: + if ((tag == 0) || ((tag & 7) == 4)) { + CHK_(ptr); + ctx->SetLastTag(tag); + goto message_done; + } + ptr = UnknownFieldParse( + tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + } // while +message_done: + _has_bits_.Or(has_bits); + return ptr; +failure: + ptr = nullptr; + goto message_done; +#undef CHK_ +} + +::PROTOBUF_NAMESPACE_ID::uint8* OnnxFlowParameter::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:onnxflow.OnnxFlowParameter) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + cached_has_bits = _has_bits_[0]; + // required .google.protobuf.Any data = 1; + if (cached_has_bits & 0x00000001u) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 1, _Internal::data(this), target, stream); + } + + // required bool requires_grad = 2; + if (cached_has_bits & 0x00000002u) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(2, this->_internal_requires_grad(), target); + } + + // required bool is_parameter = 3; + if (cached_has_bits & 0x00000004u) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBoolToArray(3, this->_internal_is_parameter(), target); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:onnxflow.OnnxFlowParameter) + return target; +} + +size_t OnnxFlowParameter::RequiredFieldsByteSizeFallback() const { +// @@protoc_insertion_point(required_fields_byte_size_fallback_start:onnxflow.OnnxFlowParameter) + size_t total_size = 0; + + if (_internal_has_data()) { + // required .google.protobuf.Any data = 1; + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *data_); + } + + if (_internal_has_requires_grad()) { + // required bool requires_grad = 2; + total_size += 1 + 1; + } + + if (_internal_has_is_parameter()) { + // required bool is_parameter = 3; + total_size += 1 + 1; + } + + return total_size; +} +size_t OnnxFlowParameter::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:onnxflow.OnnxFlowParameter) + size_t total_size = 0; + + if (((_has_bits_[0] & 0x00000007) ^ 0x00000007) == 0) { // All required fields are present. + // required .google.protobuf.Any data = 1; + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *data_); + + // required bool requires_grad = 2; + total_size += 1 + 1; + + // required bool is_parameter = 3; + total_size += 1 + 1; + + } else { + total_size += RequiredFieldsByteSizeFallback(); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void OnnxFlowParameter::CheckTypeAndMergeFrom( + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); +} + +void OnnxFlowParameter::MergeFrom(const OnnxFlowParameter& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:onnxflow.OnnxFlowParameter) + GOOGLE_DCHECK_NE(&from, this); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + cached_has_bits = from._has_bits_[0]; + if (cached_has_bits & 0x00000007u) { + if (cached_has_bits & 0x00000001u) { + _internal_mutable_data()->::PROTOBUF_NAMESPACE_ID::Any::MergeFrom(from._internal_data()); + } + if (cached_has_bits & 0x00000002u) { + requires_grad_ = from.requires_grad_; + } + if (cached_has_bits & 0x00000004u) { + is_parameter_ = from.is_parameter_; + } + _has_bits_[0] |= cached_has_bits; + } + _internal_metadata_.MergeFrom(from._internal_metadata_); +} + +void OnnxFlowParameter::CopyFrom(const OnnxFlowParameter& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:onnxflow.OnnxFlowParameter) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool OnnxFlowParameter::IsInitialized() const { + if (_Internal::MissingRequiredFields(_has_bits_)) return false; + return true; +} + +void OnnxFlowParameter::InternalSwap(OnnxFlowParameter* other) { + using std::swap; + _internal_metadata_.InternalSwap(&other->_internal_metadata_); + swap(_has_bits_[0], other->_has_bits_[0]); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(OnnxFlowParameter, is_parameter_) + + sizeof(OnnxFlowParameter::is_parameter_) + - PROTOBUF_FIELD_OFFSET(OnnxFlowParameter, data_)>( + reinterpret_cast(&data_), + reinterpret_cast(&other->data_)); +} + +std::string OnnxFlowParameter::GetTypeName() const { + return "onnxflow.OnnxFlowParameter"; +} + + +// =================================================================== + +class OnnxFlowParameters::_Internal { + public: +}; + +OnnxFlowParameters::OnnxFlowParameters(::PROTOBUF_NAMESPACE_ID::Arena* arena, + bool is_message_owned) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(arena, is_message_owned), + parameters_(arena) { + SharedCtor(); + if (!is_message_owned) { + RegisterArenaDtor(arena); + } + // @@protoc_insertion_point(arena_constructor:onnxflow.OnnxFlowParameters) +} +OnnxFlowParameters::OnnxFlowParameters(const OnnxFlowParameters& from) + : ::PROTOBUF_NAMESPACE_ID::MessageLite(), + parameters_(from.parameters_) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + // @@protoc_insertion_point(copy_constructor:onnxflow.OnnxFlowParameters) +} + +void OnnxFlowParameters::SharedCtor() { +} + +OnnxFlowParameters::~OnnxFlowParameters() { + // @@protoc_insertion_point(destructor:onnxflow.OnnxFlowParameters) + if (GetArenaForAllocation() != nullptr) return; + SharedDtor(); + _internal_metadata_.Delete(); +} + +inline void OnnxFlowParameters::SharedDtor() { + GOOGLE_DCHECK(GetArenaForAllocation() == nullptr); +} + +void OnnxFlowParameters::ArenaDtor(void* object) { + OnnxFlowParameters* _this = reinterpret_cast< OnnxFlowParameters* >(object); + (void)_this; +} +void OnnxFlowParameters::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} +void OnnxFlowParameters::SetCachedSize(int size) const { + _cached_size_.Set(size); +} + +void OnnxFlowParameters::Clear() { +// @@protoc_insertion_point(message_clear_start:onnxflow.OnnxFlowParameters) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + parameters_.Clear(); + _internal_metadata_.Clear(); +} + +const char* OnnxFlowParameters::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + switch (tag >> 3) { + // repeated .onnxflow.OnnxFlowParameter parameters = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_parameters(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else + goto handle_unusual; + continue; + default: + goto handle_unusual; + } // switch + handle_unusual: + if ((tag == 0) || ((tag & 7) == 4)) { + CHK_(ptr); + ctx->SetLastTag(tag); + goto message_done; + } + ptr = UnknownFieldParse( + tag, + _internal_metadata_.mutable_unknown_fields(), + ptr, ctx); + CHK_(ptr != nullptr); + } // while +message_done: + return ptr; +failure: + ptr = nullptr; + goto message_done; +#undef CHK_ +} + +::PROTOBUF_NAMESPACE_ID::uint8* OnnxFlowParameters::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:onnxflow.OnnxFlowParameters) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // repeated .onnxflow.OnnxFlowParameter parameters = 1; + for (unsigned int i = 0, + n = static_cast(this->_internal_parameters_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(1, this->_internal_parameters(i), target, stream); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = stream->WriteRaw(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).data(), + static_cast(_internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size()), target); + } + // @@protoc_insertion_point(serialize_to_array_end:onnxflow.OnnxFlowParameters) + return target; +} + +size_t OnnxFlowParameters::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:onnxflow.OnnxFlowParameters) + size_t total_size = 0; + + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated .onnxflow.OnnxFlowParameter parameters = 1; + total_size += 1UL * this->_internal_parameters_size(); + for (const auto& msg : this->parameters_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + total_size += _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString).size(); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void OnnxFlowParameters::CheckTypeAndMergeFrom( + const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) { + MergeFrom(*::PROTOBUF_NAMESPACE_ID::internal::DownCast( + &from)); +} + +void OnnxFlowParameters::MergeFrom(const OnnxFlowParameters& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:onnxflow.OnnxFlowParameters) + GOOGLE_DCHECK_NE(&from, this); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + parameters_.MergeFrom(from.parameters_); + _internal_metadata_.MergeFrom(from._internal_metadata_); +} + +void OnnxFlowParameters::CopyFrom(const OnnxFlowParameters& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:onnxflow.OnnxFlowParameters) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool OnnxFlowParameters::IsInitialized() const { + if (!::PROTOBUF_NAMESPACE_ID::internal::AllAreInitialized(parameters_)) return false; + return true; +} + +void OnnxFlowParameters::InternalSwap(OnnxFlowParameters* other) { + using std::swap; + _internal_metadata_.InternalSwap(&other->_internal_metadata_); + parameters_.InternalSwap(&other->parameters_); +} + +std::string OnnxFlowParameters::GetTypeName() const { + return "onnxflow.OnnxFlowParameters"; +} + + +// @@protoc_insertion_point(namespace_scope) +} // namespace onnxflow +PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::onnxflow::OnnxFlowParameter* Arena::CreateMaybeMessage< ::onnxflow::OnnxFlowParameter >(Arena* arena) { + return Arena::CreateMessageInternal< ::onnxflow::OnnxFlowParameter >(arena); +} +template<> PROTOBUF_NOINLINE ::onnxflow::OnnxFlowParameters* Arena::CreateMaybeMessage< ::onnxflow::OnnxFlowParameters >(Arena* arena) { + return Arena::CreateMessageInternal< ::onnxflow::OnnxFlowParameters >(arena); +} +PROTOBUF_NAMESPACE_CLOSE + +// @@protoc_insertion_point(global_scope) +#include diff --git a/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.h b/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.h new file mode 100644 index 0000000000..241239577f --- /dev/null +++ b/orttraining/orttraining/onnxflow/csrc/onnxflow.pb.h @@ -0,0 +1,602 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: onnxflow.proto + +#ifndef GOOGLE_PROTOBUF_INCLUDED_onnxflow_2eproto +#define GOOGLE_PROTOBUF_INCLUDED_onnxflow_2eproto + +#include +#include + +#include +#if PROTOBUF_VERSION < 3018000 +#error This file was generated by a newer version of protoc which is +#error incompatible with your Protocol Buffer headers. Please update +#error your headers. +#endif +#if 3018001 < PROTOBUF_MIN_PROTOC_VERSION +#error This file was generated by an older version of protoc which is +#error incompatible with your Protocol Buffer headers. Please +#error regenerate this file with a newer version of protoc. +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include // IWYU pragma: export +#include // IWYU pragma: export +#include +// @@protoc_insertion_point(includes) +#include +#define PROTOBUF_INTERNAL_EXPORT_onnxflow_2eproto +PROTOBUF_NAMESPACE_OPEN +namespace internal { +class AnyMetadata; +} // namespace internal +PROTOBUF_NAMESPACE_CLOSE + +// Internal implementation detail -- do not use these members. +struct TableStruct_onnxflow_2eproto { + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::AuxiliaryParseTableField aux[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[2] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; + static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; + static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[]; +}; +namespace onnxflow { +class OnnxFlowParameter; +struct OnnxFlowParameterDefaultTypeInternal; +extern OnnxFlowParameterDefaultTypeInternal _OnnxFlowParameter_default_instance_; +class OnnxFlowParameters; +struct OnnxFlowParametersDefaultTypeInternal; +extern OnnxFlowParametersDefaultTypeInternal _OnnxFlowParameters_default_instance_; +} // namespace onnxflow +PROTOBUF_NAMESPACE_OPEN +template<> ::onnxflow::OnnxFlowParameter* Arena::CreateMaybeMessage<::onnxflow::OnnxFlowParameter>(Arena*); +template<> ::onnxflow::OnnxFlowParameters* Arena::CreateMaybeMessage<::onnxflow::OnnxFlowParameters>(Arena*); +PROTOBUF_NAMESPACE_CLOSE +namespace onnxflow { + +// =================================================================== + +class OnnxFlowParameter final : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:onnxflow.OnnxFlowParameter) */ { + public: + inline OnnxFlowParameter() : OnnxFlowParameter(nullptr) {} + ~OnnxFlowParameter() override; + explicit constexpr OnnxFlowParameter(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); + + OnnxFlowParameter(const OnnxFlowParameter& from); + OnnxFlowParameter(OnnxFlowParameter&& from) noexcept + : OnnxFlowParameter() { + *this = ::std::move(from); + } + + inline OnnxFlowParameter& operator=(const OnnxFlowParameter& from) { + CopyFrom(from); + return *this; + } + inline OnnxFlowParameter& operator=(OnnxFlowParameter&& from) noexcept { + if (this == &from) return *this; + if (GetOwningArena() == from.GetOwningArena() + #ifdef PROTOBUF_FORCE_COPY_IN_MOVE + && GetOwningArena() != nullptr + #endif // !PROTOBUF_FORCE_COPY_IN_MOVE + ) { + InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); + } + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); + } + + static const OnnxFlowParameter& default_instance() { + return *internal_default_instance(); + } + static inline const OnnxFlowParameter* internal_default_instance() { + return reinterpret_cast( + &_OnnxFlowParameter_default_instance_); + } + static constexpr int kIndexInFileMessages = + 0; + + friend void swap(OnnxFlowParameter& a, OnnxFlowParameter& b) { + a.Swap(&b); + } + inline void Swap(OnnxFlowParameter* other) { + if (other == this) return; + if (GetOwningArena() == other->GetOwningArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(OnnxFlowParameter* other) { + if (other == this) return; + GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline OnnxFlowParameter* New() const final { + return new OnnxFlowParameter(); + } + + OnnxFlowParameter* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; + void CopyFrom(const OnnxFlowParameter& from); + void MergeFrom(const OnnxFlowParameter& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; + void DiscardUnknownFields(); + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + void InternalSwap(OnnxFlowParameter* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "onnxflow.OnnxFlowParameter"; + } + protected: + explicit OnnxFlowParameter(::PROTOBUF_NAMESPACE_ID::Arena* arena, + bool is_message_owned = false); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); + public: + + std::string GetTypeName() const final; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kDataFieldNumber = 1, + kRequiresGradFieldNumber = 2, + kIsParameterFieldNumber = 3, + }; + // required .google.protobuf.Any data = 1; + bool has_data() const; + private: + bool _internal_has_data() const; + public: + void clear_data(); + const ::PROTOBUF_NAMESPACE_ID::Any& data() const; + PROTOBUF_MUST_USE_RESULT ::PROTOBUF_NAMESPACE_ID::Any* release_data(); + ::PROTOBUF_NAMESPACE_ID::Any* mutable_data(); + void set_allocated_data(::PROTOBUF_NAMESPACE_ID::Any* data); + private: + const ::PROTOBUF_NAMESPACE_ID::Any& _internal_data() const; + ::PROTOBUF_NAMESPACE_ID::Any* _internal_mutable_data(); + public: + void unsafe_arena_set_allocated_data( + ::PROTOBUF_NAMESPACE_ID::Any* data); + ::PROTOBUF_NAMESPACE_ID::Any* unsafe_arena_release_data(); + + // required bool requires_grad = 2; + bool has_requires_grad() const; + private: + bool _internal_has_requires_grad() const; + public: + void clear_requires_grad(); + bool requires_grad() const; + void set_requires_grad(bool value); + private: + bool _internal_requires_grad() const; + void _internal_set_requires_grad(bool value); + public: + + // required bool is_parameter = 3; + bool has_is_parameter() const; + private: + bool _internal_has_is_parameter() const; + public: + void clear_is_parameter(); + bool is_parameter() const; + void set_is_parameter(bool value); + private: + bool _internal_is_parameter() const; + void _internal_set_is_parameter(bool value); + public: + + // @@protoc_insertion_point(class_scope:onnxflow.OnnxFlowParameter) + private: + class _Internal; + + // helper for ByteSizeLong() + size_t RequiredFieldsByteSizeFallback() const; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + ::PROTOBUF_NAMESPACE_ID::Any* data_; + bool requires_grad_; + bool is_parameter_; + friend struct ::TableStruct_onnxflow_2eproto; +}; +// ------------------------------------------------------------------- + +class OnnxFlowParameters final : + public ::PROTOBUF_NAMESPACE_ID::MessageLite /* @@protoc_insertion_point(class_definition:onnxflow.OnnxFlowParameters) */ { + public: + inline OnnxFlowParameters() : OnnxFlowParameters(nullptr) {} + ~OnnxFlowParameters() override; + explicit constexpr OnnxFlowParameters(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized); + + OnnxFlowParameters(const OnnxFlowParameters& from); + OnnxFlowParameters(OnnxFlowParameters&& from) noexcept + : OnnxFlowParameters() { + *this = ::std::move(from); + } + + inline OnnxFlowParameters& operator=(const OnnxFlowParameters& from) { + CopyFrom(from); + return *this; + } + inline OnnxFlowParameters& operator=(OnnxFlowParameters&& from) noexcept { + if (this == &from) return *this; + if (GetOwningArena() == from.GetOwningArena() + #ifdef PROTOBUF_FORCE_COPY_IN_MOVE + && GetOwningArena() != nullptr + #endif // !PROTOBUF_FORCE_COPY_IN_MOVE + ) { + InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + inline const std::string& unknown_fields() const { + return _internal_metadata_.unknown_fields(::PROTOBUF_NAMESPACE_ID::internal::GetEmptyString); + } + inline std::string* mutable_unknown_fields() { + return _internal_metadata_.mutable_unknown_fields(); + } + + static const OnnxFlowParameters& default_instance() { + return *internal_default_instance(); + } + static inline const OnnxFlowParameters* internal_default_instance() { + return reinterpret_cast( + &_OnnxFlowParameters_default_instance_); + } + static constexpr int kIndexInFileMessages = + 1; + + friend void swap(OnnxFlowParameters& a, OnnxFlowParameters& b) { + a.Swap(&b); + } + inline void Swap(OnnxFlowParameters* other) { + if (other == this) return; + if (GetOwningArena() == other->GetOwningArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(OnnxFlowParameters* other) { + if (other == this) return; + GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena()); + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline OnnxFlowParameters* New() const final { + return new OnnxFlowParameters(); + } + + OnnxFlowParameters* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CheckTypeAndMergeFrom(const ::PROTOBUF_NAMESPACE_ID::MessageLite& from) final; + void CopyFrom(const OnnxFlowParameters& from); + void MergeFrom(const OnnxFlowParameters& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; + void DiscardUnknownFields(); + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + void InternalSwap(OnnxFlowParameters* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "onnxflow.OnnxFlowParameters"; + } + protected: + explicit OnnxFlowParameters(::PROTOBUF_NAMESPACE_ID::Arena* arena, + bool is_message_owned = false); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); + public: + + std::string GetTypeName() const final; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kParametersFieldNumber = 1, + }; + // repeated .onnxflow.OnnxFlowParameter parameters = 1; + int parameters_size() const; + private: + int _internal_parameters_size() const; + public: + void clear_parameters(); + ::onnxflow::OnnxFlowParameter* mutable_parameters(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::onnxflow::OnnxFlowParameter >* + mutable_parameters(); + private: + const ::onnxflow::OnnxFlowParameter& _internal_parameters(int index) const; + ::onnxflow::OnnxFlowParameter* _internal_add_parameters(); + public: + const ::onnxflow::OnnxFlowParameter& parameters(int index) const; + ::onnxflow::OnnxFlowParameter* add_parameters(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::onnxflow::OnnxFlowParameter >& + parameters() const; + + // @@protoc_insertion_point(class_scope:onnxflow.OnnxFlowParameters) + private: + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::onnxflow::OnnxFlowParameter > parameters_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_onnxflow_2eproto; +}; +// =================================================================== + + +// =================================================================== + +#ifdef __GNUC__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif // __GNUC__ +// OnnxFlowParameter + +// required .google.protobuf.Any data = 1; +inline bool OnnxFlowParameter::_internal_has_data() const { + bool value = (_has_bits_[0] & 0x00000001u) != 0; + PROTOBUF_ASSUME(!value || data_ != nullptr); + return value; +} +inline bool OnnxFlowParameter::has_data() const { + return _internal_has_data(); +} +inline const ::PROTOBUF_NAMESPACE_ID::Any& OnnxFlowParameter::_internal_data() const { + const ::PROTOBUF_NAMESPACE_ID::Any* p = data_; + return p != nullptr ? *p : reinterpret_cast( + ::PROTOBUF_NAMESPACE_ID::_Any_default_instance_); +} +inline const ::PROTOBUF_NAMESPACE_ID::Any& OnnxFlowParameter::data() const { + // @@protoc_insertion_point(field_get:onnxflow.OnnxFlowParameter.data) + return _internal_data(); +} +inline void OnnxFlowParameter::unsafe_arena_set_allocated_data( + ::PROTOBUF_NAMESPACE_ID::Any* data) { + if (GetArenaForAllocation() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(data_); + } + data_ = data; + if (data) { + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:onnxflow.OnnxFlowParameter.data) +} +inline ::PROTOBUF_NAMESPACE_ID::Any* OnnxFlowParameter::release_data() { + _has_bits_[0] &= ~0x00000001u; + ::PROTOBUF_NAMESPACE_ID::Any* temp = data_; + data_ = nullptr; +#ifdef PROTOBUF_FORCE_COPY_IN_RELEASE + auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp); + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + if (GetArenaForAllocation() == nullptr) { delete old; } +#else // PROTOBUF_FORCE_COPY_IN_RELEASE + if (GetArenaForAllocation() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } +#endif // !PROTOBUF_FORCE_COPY_IN_RELEASE + return temp; +} +inline ::PROTOBUF_NAMESPACE_ID::Any* OnnxFlowParameter::unsafe_arena_release_data() { + // @@protoc_insertion_point(field_release:onnxflow.OnnxFlowParameter.data) + _has_bits_[0] &= ~0x00000001u; + ::PROTOBUF_NAMESPACE_ID::Any* temp = data_; + data_ = nullptr; + return temp; +} +inline ::PROTOBUF_NAMESPACE_ID::Any* OnnxFlowParameter::_internal_mutable_data() { + _has_bits_[0] |= 0x00000001u; + if (data_ == nullptr) { + auto* p = CreateMaybeMessage<::PROTOBUF_NAMESPACE_ID::Any>(GetArenaForAllocation()); + data_ = p; + } + return data_; +} +inline ::PROTOBUF_NAMESPACE_ID::Any* OnnxFlowParameter::mutable_data() { + ::PROTOBUF_NAMESPACE_ID::Any* _msg = _internal_mutable_data(); + // @@protoc_insertion_point(field_mutable:onnxflow.OnnxFlowParameter.data) + return _msg; +} +inline void OnnxFlowParameter::set_allocated_data(::PROTOBUF_NAMESPACE_ID::Any* data) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation(); + if (message_arena == nullptr) { + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(data_); + } + if (data) { + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper< + ::PROTOBUF_NAMESPACE_ID::MessageLite>::GetOwningArena( + reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(data)); + if (message_arena != submessage_arena) { + data = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, data, submessage_arena); + } + _has_bits_[0] |= 0x00000001u; + } else { + _has_bits_[0] &= ~0x00000001u; + } + data_ = data; + // @@protoc_insertion_point(field_set_allocated:onnxflow.OnnxFlowParameter.data) +} + +// required bool requires_grad = 2; +inline bool OnnxFlowParameter::_internal_has_requires_grad() const { + bool value = (_has_bits_[0] & 0x00000002u) != 0; + return value; +} +inline bool OnnxFlowParameter::has_requires_grad() const { + return _internal_has_requires_grad(); +} +inline void OnnxFlowParameter::clear_requires_grad() { + requires_grad_ = false; + _has_bits_[0] &= ~0x00000002u; +} +inline bool OnnxFlowParameter::_internal_requires_grad() const { + return requires_grad_; +} +inline bool OnnxFlowParameter::requires_grad() const { + // @@protoc_insertion_point(field_get:onnxflow.OnnxFlowParameter.requires_grad) + return _internal_requires_grad(); +} +inline void OnnxFlowParameter::_internal_set_requires_grad(bool value) { + _has_bits_[0] |= 0x00000002u; + requires_grad_ = value; +} +inline void OnnxFlowParameter::set_requires_grad(bool value) { + _internal_set_requires_grad(value); + // @@protoc_insertion_point(field_set:onnxflow.OnnxFlowParameter.requires_grad) +} + +// required bool is_parameter = 3; +inline bool OnnxFlowParameter::_internal_has_is_parameter() const { + bool value = (_has_bits_[0] & 0x00000004u) != 0; + return value; +} +inline bool OnnxFlowParameter::has_is_parameter() const { + return _internal_has_is_parameter(); +} +inline void OnnxFlowParameter::clear_is_parameter() { + is_parameter_ = false; + _has_bits_[0] &= ~0x00000004u; +} +inline bool OnnxFlowParameter::_internal_is_parameter() const { + return is_parameter_; +} +inline bool OnnxFlowParameter::is_parameter() const { + // @@protoc_insertion_point(field_get:onnxflow.OnnxFlowParameter.is_parameter) + return _internal_is_parameter(); +} +inline void OnnxFlowParameter::_internal_set_is_parameter(bool value) { + _has_bits_[0] |= 0x00000004u; + is_parameter_ = value; +} +inline void OnnxFlowParameter::set_is_parameter(bool value) { + _internal_set_is_parameter(value); + // @@protoc_insertion_point(field_set:onnxflow.OnnxFlowParameter.is_parameter) +} + +// ------------------------------------------------------------------- + +// OnnxFlowParameters + +// repeated .onnxflow.OnnxFlowParameter parameters = 1; +inline int OnnxFlowParameters::_internal_parameters_size() const { + return parameters_.size(); +} +inline int OnnxFlowParameters::parameters_size() const { + return _internal_parameters_size(); +} +inline void OnnxFlowParameters::clear_parameters() { + parameters_.Clear(); +} +inline ::onnxflow::OnnxFlowParameter* OnnxFlowParameters::mutable_parameters(int index) { + // @@protoc_insertion_point(field_mutable:onnxflow.OnnxFlowParameters.parameters) + return parameters_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::onnxflow::OnnxFlowParameter >* +OnnxFlowParameters::mutable_parameters() { + // @@protoc_insertion_point(field_mutable_list:onnxflow.OnnxFlowParameters.parameters) + return ¶meters_; +} +inline const ::onnxflow::OnnxFlowParameter& OnnxFlowParameters::_internal_parameters(int index) const { + return parameters_.Get(index); +} +inline const ::onnxflow::OnnxFlowParameter& OnnxFlowParameters::parameters(int index) const { + // @@protoc_insertion_point(field_get:onnxflow.OnnxFlowParameters.parameters) + return _internal_parameters(index); +} +inline ::onnxflow::OnnxFlowParameter* OnnxFlowParameters::_internal_add_parameters() { + return parameters_.Add(); +} +inline ::onnxflow::OnnxFlowParameter* OnnxFlowParameters::add_parameters() { + ::onnxflow::OnnxFlowParameter* _add = _internal_add_parameters(); + // @@protoc_insertion_point(field_add:onnxflow.OnnxFlowParameters.parameters) + return _add; +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::onnxflow::OnnxFlowParameter >& +OnnxFlowParameters::parameters() const { + // @@protoc_insertion_point(field_list:onnxflow.OnnxFlowParameters.parameters) + return parameters_; +} + +#ifdef __GNUC__ + #pragma GCC diagnostic pop +#endif // __GNUC__ +// ------------------------------------------------------------------- + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace onnxflow + +// @@protoc_insertion_point(global_scope) + +#include +#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_onnxflow_2eproto diff --git a/orttraining/orttraining/onnxflow/models/simple_model.onnx b/orttraining/orttraining/onnxflow/models/simple_model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..2a81d1fd28250f5609b08757588dacfc99226a7a GIT binary patch literal 680 zcmbVJO;5r=6r|9WgTS#2d0YE!P+UKryT_Hv>-|QO`Yn(rl>hb` zs627Q#rQy9 z3#|Q?*w>|d7SZ4iv;+x24Sm(b@u}LqLvb~sA>c{|7gEYnFD%4u&;+&tRg}})SEbB5 iR{@Yf%Uks17JhueJ?M(L0d1@mUgdlK8IW loss_scaler*max_norm_clip: global_gradient_norm / max_norm_clip + # else: loss_scaler*max_norm_clip + + # make the node + optimizer_node = helper.make_node("AdamOptimizer", + node_input_names, + node_output_names, + name=f"AdamOptimizer{idx}", + domain='com.microsoft', + **node_attributes) + + graph_nodes.append(optimizer_node) + + # make the graph and the model + graph = helper.make_graph(graph_nodes, 'AdamOptimizerGraph', graph_inputs, graph_outputs) + model = helper.make_model(graph, producer_name='onnxflow', + opset_imports=[helper.make_opsetid('com.microsoft', 1)]) + return model diff --git a/orttraining/orttraining/onnxflow/sample.m.cpp b/orttraining/orttraining/onnxflow/sample.m.cpp new file mode 100644 index 0000000000..4830da74f9 --- /dev/null +++ b/orttraining/orttraining/onnxflow/sample.m.cpp @@ -0,0 +1,32 @@ +#include "orttraining/onnxflow/csrc/load_parameters.h" +#include +#include +#include + +int main() +{ + std::string path_to_parameters_proto; + std::cout << "Provide the absolute path to the parameters.of file\n"; + std::cin >> path_to_parameters_proto; + std::filesystem::path path{path_to_parameters_proto}; + + auto parameters = onnxflow::load_parameters(std::filesystem::absolute(path).string()); + + std::cout << "The parameters are:\n"; + for (const auto& param : parameters.parameters()) + { + if (param.is_parameter()) + { + onnx::TensorProto tensor; + param.data().UnpackTo(&tensor); + std::cout << "<" << tensor.name() << ", requires_grad=" << (param.requires_grad() ? "True" : "False") << ">" << std::endl; + } else + { + onnx::ValueInfoProto valueinfo; + param.data().UnpackTo(&valueinfo); + std::cout << "<" << valueinfo.name() << ", requires_grad=" << (param.requires_grad() ? "True" : "False") << ">" << std::endl; + } + } + + return 0; +} \ No newline at end of file diff --git a/orttraining/orttraining/onnxflow/sample.py b/orttraining/orttraining/onnxflow/sample.py new file mode 100644 index 0000000000..6d2cef6095 --- /dev/null +++ b/orttraining/orttraining/onnxflow/sample.py @@ -0,0 +1,36 @@ +from onnxflow import TrainingGraph, Graph +import onnxflow +import onnx + +class MyGraph(TrainingGraph): + def __init__(self, base_model): + super(MyGraph, self).__init__() + self.loss = onnxflow.loss.MSELoss() + self.base_model = base_model + + def build(self): + outputs = self.base_model.graph.output + lossful_graph = self.loss(self.base_model, outputs[0].name) + return lossful_graph + +onnxfile = 'models/simple_model.onnx' +model = onnx.load(onnxfile) + +graph = MyGraph(model) + +# remove in case of any model other than simple_model.onnx +graph.requires_grad('_original_module.fc1.weight') +graph.requires_grad('_original_module.fc1.bias') +graph.requires_grad('_original_module.fc2.weight') +graph.requires_grad('_original_module.fc2.bias') + +gradient_graph = graph() + +parameters = graph.parameters() +onnxflow.save(parameters, 'parameters.of') + +optimizer = onnxflow.optim.AdamW() +optimizer_graph = optimizer(gradient_graph) + +onnx.save(gradient_graph, "gradient_graph.onnx") +onnx.save(optimizer_graph, "optimizer_graph.onnx") diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index dcf8913925..17b71539fc 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -584,6 +584,11 @@ def parse_arguments(): parser.add_argument('--eager_customop_header', default=None, help='Header containing custom op definitions for eager mode.') + # on device training + parser.add_argument( + "--build_on_device_training", action='store_true', + help="Build on device training.") + parser.add_argument( "--enable_external_custom_op_schemas", action='store_true', help="Enable registering user defined custom operation schemas at shared library load time.\ @@ -860,6 +865,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_ENABLE_WEBASSEMBLY_DEBUG_INFO=" + ("ON" if args.enable_wasm_debug_info else "OFF"), "-Donnxruntime_ENABLE_WEBASSEMBLY_PROFILING=" + ("ON" if args.enable_wasm_profiling else "OFF"), "-Donnxruntime_ENABLE_EAGER_MODE=" + ("ON" if args.build_eager_mode else "OFF"), + "-Donnxruntime_ENABLE_ON_DEVICE_TRAINING=" + ("ON" if args.build_on_device_training else "OFF"), "-Donnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS=" + ("ON" if args.enable_external_custom_op_schemas else "OFF"), "-Donnxruntime_NVCC_THREADS=" + str(args.parallel),