diff --git a/cgmanifest.json b/cgmanifest.json index 8c01bf408c..9332ee366a 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -450,7 +450,7 @@ { "component": { "git": { - "commitHash": "006df6bff45dac59d378609fe85f6736a901ee93", + "commitHash": "4df80d5865a9d4e97f6d0b9304d4316115a04d9e", "repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git" }, "type": "git" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 59ca5465e1..2a0d1b03d5 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -55,7 +55,7 @@ option(onnxruntime_USE_NNAPI "Build with DNNLibrary for Android NNAPI support" O option(onnxruntime_USE_DNNL "Build with DNNL support" OFF) option(onnxruntime_USE_MKLML "Build DNNL with MKL-ML binary dependency" OFF) option(onnxruntime_USE_GEMMLOWP "Build with gemmlowp for quantized gemm" OFF) -option(onnxruntime_USE_AUTOML "Build AutoML support" ON) +option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" ON) option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF) option(onnxruntime_USE_OPENBLAS "Use openblas" OFF) option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF) @@ -709,8 +709,8 @@ include(onnxruntime_optimizer.cmake) include(onnxruntime_session.cmake) include(onnxruntime_mlas.cmake) -if(onnxruntime_USE_AUTOML) - add_definitions(-DMICROSOFT_AUTOML) +if(onnxruntime_USE_FEATURIZERS) + add_definitions(-DML_FEATURIZERS) # Fetch and build featurizers include(external/featurizers.cmake) endif() diff --git a/cmake/external/featurizers.cmake b/cmake/external/featurizers.cmake index ba23b4fdef..b9e66570cd 100644 --- a/cmake/external/featurizers.cmake +++ b/cmake/external/featurizers.cmake @@ -3,7 +3,7 @@ # This source code should not depend on the onnxruntime and may be built independently set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git") -set(featurizers_TAG "006df6bff45dac59d378609fe85f6736a901ee93") +set(featurizers_TAG "4df80d5865a9d4e97f6d0b9304d4316115a04d9e") set(featurizers_pref FeaturizersLibrary) set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref}) @@ -41,45 +41,19 @@ else() ) endif() -add_library(automl_featurizers STATIC IMPORTED) -add_dependencies(automl_featurizers featurizers_lib) -target_include_directories(automl_featurizers INTERFACE ${featurizers_ROOT}/src) +add_library(onnxruntime_featurizers STATIC IMPORTED) +add_dependencies(onnxruntime_featurizers featurizers_lib) +target_include_directories(onnxruntime_featurizers INTERFACE ${featurizers_ROOT}/src) if(MSVC) - set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION + set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/${CMAKE_BUILD_TYPE}/FeaturizersCode.lib) else() - set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION + set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/libFeaturizersCode.a) endif() if (WIN32) # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. - set_target_properties(automl_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) + set_target_properties(onnxruntime_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) endif() - -# Build this in CentOS -# foreach(_test_name IN ITEMS - # CatImputerFeaturizer_UnitTests - # DateTimeFeaturizer_UnitTests - # HashOneHotVectorizerFeaturizer_UnitTests - # ImputationMarkerFeaturizer_UnitTests - # LabelEncoderFeaturizer_UnitTests - # MaxAbsScalarFeaturizer_UnitTests - # MinMaxScalarFeaturizer_UnitTests - # MissingDummiesFeaturizer_UnitTests - # OneHotEncoderFeaturizer_UnitTests - # RobustScalarFeaturizer_UnitTests - # SampleAddFeaturizer_UnitTest - # StringFeaturizer_UnitTest - # Structs_UnitTest - # TimeSeriesImputerFeaturizer_UnitTest -# ) - # add_executable(${_test_name} ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp) - # add_dependencies(${_test_name} automl_featurizers) - # target_include_directories(${_test_name} PRIVATE ${featurizers_ROOT}/src) - # target_link_libraries(${_test_name} automl_featurizers) - # list(APPEND featurizers_TEST_SRC ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp) -# endforeach() - -# source_group(TREE ${featurizers_ROOT}/src/Featurizers/UnitTests FILES ${featurizers_TEST_SRC}) diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake index edcf18ae04..ae11ad51e0 100644 --- a/cmake/onnxruntime_graph.cmake +++ b/cmake/onnxruntime_graph.cmake @@ -14,12 +14,12 @@ if (onnxruntime_DISABLE_CONTRIB_OPS) ) endif() -if(NOT onnxruntime_USE_AUTOML) - file(GLOB_RECURSE automl_to_remove_graph_src - "${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.h" - "${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.cc" +if(NOT onnxruntime_USE_FEATURIZERS) + file(GLOB_RECURSE featurizers_to_remove_graph_src + "${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.h" + "${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.cc" ) - foreach(I in ${automl_to_remove_graph_src}) + foreach(I in ${featurizers_to_remove_graph_src}) list(REMOVE_ITEM onnxruntime_graph_src ${I}) endforeach() endif() diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 7549e2b5e2..c480a8ddb8 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -25,14 +25,11 @@ file(GLOB_RECURSE onnxruntime_cuda_contrib_ops_cu_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/contrib_ops/cuda/*.cuh" ) -file(GLOB onnxruntime_cpu_automl_cc_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.cc" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_types.h" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_types.cc" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_featurizers.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.cc" +file(GLOB onnxruntime_cpu_featurizers_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.h" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.cc" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.h" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.cc" ) file(GLOB onnxruntime_providers_common_srcs CONFIGURE_DEPENDS @@ -87,18 +84,18 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS) list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_contrib_ops_srcs}) endif() -if (onnxruntime_USE_AUTOML) - source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_automl_cc_srcs}) - list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_automl_cc_srcs}) +if (onnxruntime_USE_FEATURIZERS) + source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_featurizers_cc_srcs}) + list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_featurizers_cc_srcs}) endif() add_library(onnxruntime_providers ${onnxruntime_providers_src}) onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) -if (onnxruntime_USE_AUTOML) - add_dependencies(onnxruntime_providers automl_featurizers) - onnxruntime_add_include_to_target(onnxruntime_providers automl_featurizers) - target_link_libraries(onnxruntime_providers automl_featurizers) +if (onnxruntime_USE_FEATURIZERS) + add_dependencies(onnxruntime_providers onnxruntime_featurizers) + onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_featurizers) + target_link_libraries(onnxruntime_providers onnxruntime_featurizers) endif() if(HAS_DEPRECATED_COPY) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 115776cffd..44dbd0017c 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -129,10 +129,10 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS) "${TEST_SRC_DIR}/contrib_ops/*.cc") endif() -if(onnxruntime_USE_AUTOML) +if(onnxruntime_USE_FEATURIZERS) list(APPEND onnxruntime_test_providers_src_patterns - "${TEST_SRC_DIR}/automl_ops/*.h" - "${TEST_SRC_DIR}/automl_ops/*.cc") + "${TEST_SRC_DIR}/featurizers_ops/*.h" + "${TEST_SRC_DIR}/featurizers_ops/*.cc") endif() file(GLOB onnxruntime_test_providers_src CONFIGURE_DEPENDS @@ -218,8 +218,8 @@ if(onnxruntime_USE_NNAPI) list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi) endif() -if(onnxruntime_USE_AUTOML) - list(APPEND onnxruntime_test_providers_dependencies automl_featurizers) +if(onnxruntime_USE_FEATURIZERS) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_featurizers) endif() if(onnxruntime_USE_DML) diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h index 323fd19943..dda7578706 100644 --- a/include/onnxruntime/core/framework/op_kernel.h +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -246,10 +246,10 @@ template KernelCreateInfo BuildKernelCreateInfo(); } // namespace contrib -namespace automl { +namespace featurizers { template KernelCreateInfo BuildKernelCreateInfo(); -} // namespace automl +} // namespace featurizers namespace contrib { namespace cuda { diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index 26fc15d3dd..a4df1eb7b9 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -19,7 +19,7 @@ constexpr const char* kOnnxDomainAlias = "ai.onnx"; constexpr const char* kMLDomain = "ai.onnx.ml"; constexpr const char* kMSDomain = "com.microsoft"; constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc"; -constexpr const char* kMSAutoMLDomain = "com.microsoft.automl"; +constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers"; constexpr const char* kMSDmlDomain = "com.microsoft.dml"; constexpr const char* kNGraphDomain = "com.intel.ai"; constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider"; diff --git a/include/onnxruntime/core/session/automl_data_containers.h b/include/onnxruntime/core/session/automl_data_containers.h index e7f20f77b4..249e42a483 100644 --- a/include/onnxruntime/core/session/automl_data_containers.h +++ b/include/onnxruntime/core/session/automl_data_containers.h @@ -11,7 +11,7 @@ extern "C" { #endif // This structure is used to initialize and read -// OrtValue of opaque(com.microsoft.automl,DateTimeFeaturizer_TimePoint) +// OrtValue of opaque(com.microsoft.featurizers,DateTimeFeaturizer_TimePoint) struct DateTimeFeaturizerTimePointData { int32_t year; uint8_t month; diff --git a/onnxruntime/automl_ops/cpu/datetime_transformer.cc b/onnxruntime/automl_ops/cpu/datetime_transformer.cc deleted file mode 100644 index db3fe6ab20..0000000000 --- a/onnxruntime/automl_ops/cpu/datetime_transformer.cc +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/common/common.h" -#include "core/framework/data_types.h" -#include "core/framework/op_kernel.h" - -#include "Featurizers/DateTimeFeaturizer.h" - -namespace featurizers = Microsoft::Featurizer::Featurizers; - -namespace onnxruntime { -namespace automl { - -class DateTimeTransformer final : public OpKernel { - public: - explicit DateTimeTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { - // Create the transformer - featurizers::DateTimeTransformer transformer( - [ctx]() { - const auto* state_tensor(ctx->Input(0)); - const uint8_t* const state_data(state_tensor->Data()); - - Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); - return featurizers::DateTimeTransformer(archive); - }()); - - // Get the input - const auto* input_tensor(ctx->Input(1)); - const int64_t* input_data(input_tensor->Data()); - - // Prepare the output - Tensor* year_tensor(ctx->Output(0, input_tensor->Shape())); - Tensor* month_tensor(ctx->Output(1, input_tensor->Shape())); - Tensor* day_tensor(ctx->Output(2, input_tensor->Shape())); - Tensor* hour_tensor(ctx->Output(3, input_tensor->Shape())); - Tensor* minute_tensor(ctx->Output(4, input_tensor->Shape())); - Tensor* second_tensor(ctx->Output(5, input_tensor->Shape())); - Tensor* amPm_tensor(ctx->Output(6, input_tensor->Shape())); - Tensor* hour12_tensor(ctx->Output(7, input_tensor->Shape())); - Tensor* dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape())); - Tensor* dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape())); - Tensor* dayOfYear_tensor(ctx->Output(10, input_tensor->Shape())); - Tensor* weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape())); - Tensor* quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape())); - Tensor* halfOfYear_tensor(ctx->Output(13, input_tensor->Shape())); - Tensor* weekIso_tensor(ctx->Output(14, input_tensor->Shape())); - Tensor* yearIso_tensor(ctx->Output(15, input_tensor->Shape())); - Tensor* monthLabel_tensor(ctx->Output(16, input_tensor->Shape())); - Tensor* amPmLabel_tensor(ctx->Output(17, input_tensor->Shape())); - Tensor* dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape())); - Tensor* holidayName_tensor(ctx->Output(19, input_tensor->Shape())); - Tensor* isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape())); - - int32_t* year_data(year_tensor->MutableData()); - uint8_t* month_data(month_tensor->MutableData()); - uint8_t* day_data(day_tensor->MutableData()); - uint8_t* hour_data(hour_tensor->MutableData()); - uint8_t* minute_data(minute_tensor->MutableData()); - uint8_t* second_data(second_tensor->MutableData()); - uint8_t* amPm_data(amPm_tensor->MutableData()); - uint8_t* hour12_data(hour12_tensor->MutableData()); - uint8_t* dayOfWeek_data(dayOfWeek_tensor->MutableData()); - uint8_t* dayOfQuarter_data(dayOfQuarter_tensor->MutableData()); - uint16_t* dayOfYear_data(dayOfYear_tensor->MutableData()); - uint16_t* weekOfMonth_data(weekOfMonth_tensor->MutableData()); - uint8_t* quarterOfYear_data(quarterOfYear_tensor->MutableData()); - uint8_t* halfOfYear_data(halfOfYear_tensor->MutableData()); - uint8_t* weekIso_data(weekIso_tensor->MutableData()); - int32_t* yearIso_data(yearIso_tensor->MutableData()); - std::string* monthLabel_data(monthLabel_tensor->MutableData()); - std::string* amPmLabel_data(amPmLabel_tensor->MutableData()); - std::string* dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData()); - std::string* holidayName_data(holidayName_tensor->MutableData()); - uint8_t* isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData()); - - // Execute - int64_t const length = input_tensor->Shape().Size(); - - for (int64_t i = 0; i < length; ++i) { - auto result(transformer.execute(input_data[i])); - - year_data[i] = std::move(result.year); - month_data[i] = std::move(result.month); - day_data[i] = std::move(result.day); - hour_data[i] = std::move(result.hour); - minute_data[i] = std::move(result.minute); - second_data[i] = std::move(result.second); - amPm_data[i] = std::move(result.amPm); - hour12_data[i] = std::move(result.hour12); - dayOfWeek_data[i] = std::move(result.dayOfWeek); - dayOfQuarter_data[i] = std::move(result.dayOfQuarter); - dayOfYear_data[i] = std::move(result.dayOfYear); - weekOfMonth_data[i] = std::move(result.weekOfMonth); - quarterOfYear_data[i] = std::move(result.quarterOfYear); - halfOfYear_data[i] = std::move(result.halfOfYear); - weekIso_data[i] = std::move(result.weekIso); - yearIso_data[i] = std::move(result.yearIso); - monthLabel_data[i] = std::move(result.monthLabel); - amPmLabel_data[i] = std::move(result.amPmLabel); - dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel); - holidayName_data[i] = std::move(result.holidayName); - isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff); - } - - return Status::OK(); - } -}; - -ONNX_OPERATOR_KERNEL_EX( - DateTimeTransformer, - kMSAutoMLDomain, - 1, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT0", DataTypeImpl::GetTensorType()) - .TypeConstraint("InputT1", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT0", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT1", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT2", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT3", DataTypeImpl::GetTensorType()), - DateTimeTransformer); - -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/automl_ops/cpu_automl_kernels.cc b/onnxruntime/automl_ops/cpu_automl_kernels.cc deleted file mode 100644 index e2a0e41ab9..0000000000 --- a/onnxruntime/automl_ops/cpu_automl_kernels.cc +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "automl_ops/cpu_automl_kernels.h" -#include "core/graph/constants.h" -#include "core/framework/data_types.h" - -namespace onnxruntime { -namespace automl { - -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSAutoMLDomain, 1, DateTimeTransformer); - -Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) { - static const BuildKernelCreateInfoFn function_table[] = { - // add more kernels here - BuildKernelCreateInfo - }; - - for (auto& function_table_entry : function_table) { - ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); - } - return Status::OK(); -} - -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/automl_ops/automl_defs.cc b/onnxruntime/core/graph/automl_ops/automl_defs.cc deleted file mode 100644 index 38d8708b37..0000000000 --- a/onnxruntime/core/graph/automl_ops/automl_defs.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/graph/constants.h" -#include "core/graph/automl_ops/automl_defs.h" -#include "core/graph/op.h" -#include "onnx/defs/schema.h" -#include "onnx/defs/shape_inference.h" - -namespace onnxruntime { -namespace automl { -using ONNX_NAMESPACE::AttributeProto; -using ONNX_NAMESPACE::OpSchema; -using ONNX_NAMESPACE::OPTIONAL; - -static const char* DateTimeTransformer_ver1_doc = R"DOC( - Extracts various datetime-related values from a UTC time_point. - - C++-style pseudo signature: - TimePoint execute(std::chron::system_clock::time_point const &value); - - Examples: - Given a time_point 'value' representing "November 17, 1976 12:27:04PM": - - "November 17, 1976 12:27:04PM" => { - "year": 1976, - "month": 11, - "day": 17, - "hour": 12, - "minute": 27, - "second": 04, - "amPm": 2, // PM - "hour12": 12, - "dayOfWeek": 3, // Wednesday - "dayOfQuarter": 48, - "dayOfYear": 321, - "weekOfMonth": 2, - "quarterOfYear": 4, - "halfOfYear": 2, - "weekIso": 47, - "yearIso": 1976, - "monthLabel": "November", - "amPmLabel": "pm", - "dayOfWeekLabel": "Wednesday", - "holidayName": "", - "isPaidTimeOff": 0 - } - )DOC"; - -void RegisterAutoMLSchemas() { - MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer) - .SinceVersion(1) - .SetDomain(kMSAutoMLDomain) - .SetDoc(DateTimeTransformer_ver1_doc) - .Input( - 0, - "State", - "State generated during training that is used for prediction", - "InputT0") - .Input( - 1, - "Input", - "The input represents a number of seconds passed since the epoch, suitable to properly construct" - "an instance of std::chrono::system_clock::time_point", - "InputT1") - .Output(0, "year", "No information available", "OutputT0") - .Output(1, "month", "No information available", "OutputT1") - .Output(2, "day", "No information available", "OutputT1") - .Output(3, "hour", "No information available", "OutputT1") - .Output(4, "minute", "No information available", "OutputT1") - .Output(5, "second", "No information available", "OutputT1") - .Output(6, "amPm", "No information available", "OutputT1") - .Output(7, "hour12", "No information available", "OutputT1") - .Output(8, "dayOfWeek", "No information available", "OutputT1") - .Output(9, "dayOfQuarter", "No information available", "OutputT1") - .Output(10, "dayOfYear", "No information available", "OutputT2") - .Output(11, "weekOfMonth", "No information available", "OutputT2") - .Output(12, "quarterOfYear", "No information available", "OutputT1") - .Output(13, "halfOfYear", "No information available", "OutputT1") - .Output(14, "weekIso", "No information available", "OutputT1") - .Output(15, "yearIso", "No information available", "OutputT0") - .Output(16, "monthLabel", "No information available", "OutputT3") - .Output(17, "amPmLabel", "No information available", "OutputT3") - .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") - .Output(19, "holidayName", "No information available", "OutputT3") - .Output(20, "isPaidTimeOff", "No information available", "OutputT1") - .TypeConstraint( - "InputT0", - {"tensor(uint8)"}, - "No information is available") - .TypeConstraint( - "InputT1", - {"tensor(int64)"}, - "No information is available") - .TypeConstraint( - "OutputT0", - {"tensor(int32)"}, - "No information is available") - .TypeConstraint( - "OutputT1", - {"tensor(uint8)"}, - "No information is available") - .TypeConstraint( - "OutputT2", - {"tensor(uint16)"}, - "No information is available") - .TypeConstraint( - "OutputT3", - {"tensor(string)"}, - "No information is available") - .TypeAndShapeInferenceFunction( - [](ONNX_NAMESPACE::InferenceContext& ctx) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { - *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); - } - }); -} -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/automl_ops/automl_defs.h b/onnxruntime/core/graph/automl_ops/automl_defs.h deleted file mode 100644 index b1a37366c3..0000000000 --- a/onnxruntime/core/graph/automl_ops/automl_defs.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/graph/onnx_protobuf.h" - -namespace onnxruntime { -namespace automl { -#define MS_AUTOML_OPERATOR_SCHEMA(name) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \ - static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ - op_schema_register_once##name##Counter) ONNX_UNUSED = \ - ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__) - -#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \ - static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ - op_schema_register_once##name##Counter) ONNX_UNUSED = \ - schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)) - -void RegisterAutoMLSchemas(); -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc new file mode 100644 index 0000000000..7c820dc6c2 --- /dev/null +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/constants.h" +#include "core/graph/featurizers_ops/featurizers_defs.h" +#include "core/graph/op.h" + +#include "onnx/defs/schema.h" +#include "onnx/defs/shape_inference.h" + +#define MS_AUTOML_OPERATOR_SCHEMA(name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name) +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) + +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__) + +#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func) +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) + +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)) + +namespace onnxruntime { +namespace featurizers { + +using ONNX_NAMESPACE::AttributeProto; +using ONNX_NAMESPACE::OpSchema; +using ONNX_NAMESPACE::OPTIONAL; + +// Forward declarations +static void RegisterCatImputerFeaturizerVer1(); +static void RegisterDateTimeFeaturizerVer1(); +static void RegisterMaxAbsScalarFeaturizerVer1(); +static void RegisterStringFeaturizerVer1(); + +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +void RegisterAutoMLSchemas() { + RegisterCatImputerFeaturizerVer1(); + RegisterDateTimeFeaturizerVer1(); + RegisterMaxAbsScalarFeaturizerVer1(); + RegisterStringFeaturizerVer1(); +} + +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +void RegisterCatImputerFeaturizerVer1() { + static const char * doc = R"DOC( + Imputes (populates) values with the mode (most common value) encountered during + training. This featurizer supports float and double for most (if not all) frameworks + due to the existance of NaN in those types. Other types require 'optional' support + within the host frameworks and programming languages. + + C++-style pseudo signature: + std::float_t execute(std::float_t const &value); + std::double_t execute(std::double_t const &value); + template T execute(std::optional const &value); + + Examples (where 55.5 is the mode value): + execute(1.0) -> 1.0 + execute(NaN) -> 55.5 + execute(2.0) -> 2.0 + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(CatImputerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "T" + ) + .Output( + 0, + "Output", + "No information is available", + "T" + ) + .TypeConstraint( + "T", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 1, 0); + if (!hasNInputShapes(ctx, 1)) { + return; + } + propagateShapeFromInputToOutput(ctx, 1, 0); + } + ) + ; +} + +void RegisterDateTimeFeaturizerVer1() { + static const char * doc = R"DOC( + Extracts various datetime-related values from a UTC time_point. + + C++-style pseudo signature: + TimePoint execute(std::chron::system_clock::time_point const &value); + + Examples: + Given a time_point 'value' representing "November 17, 1976 12:27:04PM": + + "November 17, 1976 12:27:04PM" => { + "year": 1976, + "month": 11, + "day": 17, + "hour": 12, + "minute": 27, + "second": 04, + "amPm": 2, // PM + "hour12": 12, + "dayOfWeek": 3, // Wednesday + "dayOfQuarter": 48, + "dayOfYear": 321, + "weekOfMonth": 2, + "quarterOfYear": 4, + "halfOfYear": 2, + "weekIso": 47, + "yearIso": 1976, + "monthLabel": "November", + "amPmLabel": "pm", + "dayOfWeekLabel": "Wednesday", + "holidayName": "", + "isPaidTimeOff": 0 + } + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "tensor(int64)" + ) + .Output(0, "year", "No information available", "OutputT0") + .Output(1, "month", "No information available", "OutputT1") + .Output(2, "day", "No information available", "OutputT1") + .Output(3, "hour", "No information available", "OutputT1") + .Output(4, "minute", "No information available", "OutputT1") + .Output(5, "second", "No information available", "OutputT1") + .Output(6, "amPm", "No information available", "OutputT1") + .Output(7, "hour12", "No information available", "OutputT1") + .Output(8, "dayOfWeek", "No information available", "OutputT1") + .Output(9, "dayOfQuarter", "No information available", "OutputT1") + .Output(10, "dayOfYear", "No information available", "OutputT2") + .Output(11, "weekOfMonth", "No information available", "OutputT2") + .Output(12, "quarterOfYear", "No information available", "OutputT1") + .Output(13, "halfOfYear", "No information available", "OutputT1") + .Output(14, "weekIso", "No information available", "OutputT1") + .Output(15, "yearIso", "No information available", "OutputT0") + .Output(16, "monthLabel", "No information available", "OutputT3") + .Output(17, "amPmLabel", "No information available", "OutputT3") + .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") + .Output(19, "holidayName", "No information available", "OutputT3") + .Output(20, "isPaidTimeOff", "No information available", "OutputT1") + .TypeConstraint( + "OutputT0", + {"tensor(int32)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT1", + {"tensor(uint8)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT2", + {"tensor(uint16)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT3", + {"tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext &ctx) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); + ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); + ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); + ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); + ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + + for(size_t i = 0; i < ctx.getNumOutputs(); ++i) { + *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + } + ) + ; +} + +void RegisterMaxAbsScalarFeaturizerVer1() { + static const char * doc = R"DOC( + Scales input based on the maximum absolute value of all data encountered during training. + + C++-style pseudo signature: + std::float_t execute(std::uint16_t value); + std::double_t execute(std::uint32_t value); + + Examples: + Given a training set of [1.0, -2.0, 3.0, -4.0], where 4.0 is the absolute value of the + maximum value encountered... + + execute(1.0) -> 1.0 / 4.0 + execute(-4.0) -> -4.0 / 4.0 + execute(100.0) -> 100 / 4.0 + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(MaxAbsScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "InputT" + ) + .Output( + 0, + "Output", + "No information is available", + "OutputT" + ) + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT", + {"tensor(float)", "tensor(double)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + auto input_elem_type = ctx.getInputType(1)->tensor_type().elem_type(); + if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE); + } + + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + ) + ; +} + +void RegisterStringFeaturizerVer1() { + static const char * doc = R"DOC( + Converts the input into a string representation based on the input's type. + + C++-style pseudo signature: + template std::string execute(T const &value); + + Examples: + execute(1) -> "1" + execute(3.14) -> "3.14" + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(StringTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "InputT" + ) + .Output( + 0, + "Output", + "No information is available", + "tensor(string)" + ) + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 0); + + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + ) + ; +} + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h new file mode 100644 index 0000000000..d1323c0b3c --- /dev/null +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace featurizers { + +void RegisterAutoMLSchemas(void); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index d56ab62fcb..77e7370dfa 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -9,8 +9,8 @@ #include "contrib_ops/cpu_contrib_kernels.h" #endif -#ifdef MICROSOFT_AUTOML -#include "automl_ops/cpu_automl_kernels.h" +#ifdef ML_FEATURIZERS +#include "featurizers_ops/cpu_featurizers_kernels.h" #endif #include "core/framework/compute_capability.h" @@ -1270,8 +1270,8 @@ static Status RegisterCPUKernels(KernelRegistry& kernel_registry) { #ifndef DISABLE_CONTRIB_OPS ORT_RETURN_IF_ERROR(::onnxruntime::contrib::RegisterCpuContribKernels(kernel_registry)); #endif -#ifdef MICROSOFT_AUTOML - ORT_RETURN_IF_ERROR(::onnxruntime::automl::RegisterCpuAutoMLKernels(kernel_registry)); +#ifdef ML_FEATURIZERS + ORT_RETURN_IF_ERROR(::onnxruntime::featurizers::RegisterCpuAutoMLKernels(kernel_registry)); #endif return Status::OK(); } diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index da0281e643..d133b9d997 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -10,8 +10,8 @@ #ifndef DISABLE_CONTRIB_OPS #include "core/graph/contrib_ops/contrib_defs.h" #endif -#ifdef MICROSOFT_AUTOML -#include "core/graph/automl_ops/automl_defs.h" +#ifdef ML_FEATURIZERS +#include "core/graph/featurizers_ops/featurizers_defs.h" #endif #ifdef USE_DML #include "core/graph/dml_ops/dml_defs.h" @@ -45,7 +45,7 @@ Status Environment::Initialize() { std::call_once(schemaRegistrationOnceFlag, []() { ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 1); ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSNchwcDomain, 1, 1); - ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSAutoMLDomain, 1, 1); + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSFeaturizersDomain, 1, 1); #ifdef USE_DML ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDmlDomain, 1, 1); #endif @@ -54,8 +54,8 @@ Status Environment::Initialize() { #ifndef DISABLE_CONTRIB_OPS contrib::RegisterContribSchemas(); #endif -#ifdef MICROSOFT_AUTOML - automl::RegisterAutoMLSchemas(); +#ifdef ML_FEATURIZERS + featurizers::RegisterAutoMLSchemas(); #endif #ifdef USE_DML dml::RegisterDmlSchemas(); diff --git a/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc b/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc new file mode 100644 index 0000000000..0d4daa51ad --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/CatImputerFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template +struct OutputTypeMapper {}; +template <> +struct OutputTypeMapper { using type = float_t; }; +template <> +struct OutputTypeMapper { using type = double_t; }; +template <> +struct OutputTypeMapper { using type = std::string; }; + +inline float_t const& PreprocessOptional(float_t const& value) { return value; } +inline double_t const& PreprocessOptional(double_t const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +class CatImputerTransformer final : public OpKernel { + public: + explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + // Create the transformer + feat::CatImputerTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::CatImputerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const T* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type* output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + string, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc b/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc new file mode 100644 index 0000000000..013222154c --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/DateTimeFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +class DateTimeTransformer final : public OpKernel { +public: + explicit DateTimeTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::DateTimeTransformer transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::DateTimeTransformer(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const std::int64_t * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * year_tensor(ctx->Output(0, input_tensor->Shape())); + Tensor * month_tensor(ctx->Output(1, input_tensor->Shape())); + Tensor * day_tensor(ctx->Output(2, input_tensor->Shape())); + Tensor * hour_tensor(ctx->Output(3, input_tensor->Shape())); + Tensor * minute_tensor(ctx->Output(4, input_tensor->Shape())); + Tensor * second_tensor(ctx->Output(5, input_tensor->Shape())); + Tensor * amPm_tensor(ctx->Output(6, input_tensor->Shape())); + Tensor * hour12_tensor(ctx->Output(7, input_tensor->Shape())); + Tensor * dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape())); + Tensor * dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape())); + Tensor * dayOfYear_tensor(ctx->Output(10, input_tensor->Shape())); + Tensor * weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape())); + Tensor * quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape())); + Tensor * halfOfYear_tensor(ctx->Output(13, input_tensor->Shape())); + Tensor * weekIso_tensor(ctx->Output(14, input_tensor->Shape())); + Tensor * yearIso_tensor(ctx->Output(15, input_tensor->Shape())); + Tensor * monthLabel_tensor(ctx->Output(16, input_tensor->Shape())); + Tensor * amPmLabel_tensor(ctx->Output(17, input_tensor->Shape())); + Tensor * dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape())); + Tensor * holidayName_tensor(ctx->Output(19, input_tensor->Shape())); + Tensor * isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape())); + + int32_t * year_data(year_tensor->MutableData()); + uint8_t * month_data(month_tensor->MutableData()); + uint8_t * day_data(day_tensor->MutableData()); + uint8_t * hour_data(hour_tensor->MutableData()); + uint8_t * minute_data(minute_tensor->MutableData()); + uint8_t * second_data(second_tensor->MutableData()); + uint8_t * amPm_data(amPm_tensor->MutableData()); + uint8_t * hour12_data(hour12_tensor->MutableData()); + uint8_t * dayOfWeek_data(dayOfWeek_tensor->MutableData()); + uint8_t * dayOfQuarter_data(dayOfQuarter_tensor->MutableData()); + uint16_t * dayOfYear_data(dayOfYear_tensor->MutableData()); + uint16_t * weekOfMonth_data(weekOfMonth_tensor->MutableData()); + uint8_t * quarterOfYear_data(quarterOfYear_tensor->MutableData()); + uint8_t * halfOfYear_data(halfOfYear_tensor->MutableData()); + uint8_t * weekIso_data(weekIso_tensor->MutableData()); + int32_t * yearIso_data(yearIso_tensor->MutableData()); + std::string * monthLabel_data(monthLabel_tensor->MutableData()); + std::string * amPmLabel_data(amPmLabel_tensor->MutableData()); + std::string * dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData()); + std::string * holidayName_data(holidayName_tensor->MutableData()); + uint8_t * isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + auto result(transformer.execute(input_data[i])); + + year_data[i] = std::move(result.year); + month_data[i] = std::move(result.month); + day_data[i] = std::move(result.day); + hour_data[i] = std::move(result.hour); + minute_data[i] = std::move(result.minute); + second_data[i] = std::move(result.second); + amPm_data[i] = std::move(result.amPm); + hour12_data[i] = std::move(result.hour12); + dayOfWeek_data[i] = std::move(result.dayOfWeek); + dayOfQuarter_data[i] = std::move(result.dayOfQuarter); + dayOfYear_data[i] = std::move(result.dayOfYear); + weekOfMonth_data[i] = std::move(result.weekOfMonth); + quarterOfYear_data[i] = std::move(result.quarterOfYear); + halfOfYear_data[i] = std::move(result.halfOfYear); + weekIso_data[i] = std::move(result.weekIso); + yearIso_data[i] = std::move(result.yearIso); + monthLabel_data[i] = std::move(result.monthLabel); + amPmLabel_data[i] = std::move(result.amPmLabel); + dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel); + holidayName_data[i] = std::move(result.holidayName); + isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + DateTimeTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("OutputT0", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT1", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT2", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT3", DataTypeImpl::GetTensorType()), + DateTimeTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc b/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc new file mode 100644 index 0000000000..3ad2ad5b8b --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MaxAbsScalarFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template struct OutputTypeMapper {}; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; + +template +class MaxAbsScalarTransformer final : public OpKernel { +public: + explicit MaxAbsScalarTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::MaxAbsScalarTransformer::type> transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::MaxAbsScalarTransformer::type>(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type * output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/StringTransformer.cc b/onnxruntime/featurizers_ops/cpu/StringTransformer.cc new file mode 100644 index 0000000000..48a5ca424d --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/StringTransformer.cc @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/StringFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template +class StringTransformer final : public OpKernel { +public: + explicit StringTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::StringTransformer transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::StringTransformer(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * output_tensor(ctx->Output(0, input_tensor->Shape())); + std::string * output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + bool, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + string, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc new file mode 100644 index 0000000000..0e11490b7d --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "featurizers_ops/cpu_featurizers_kernels.h" + +#include "core/graph/constants.h" +#include "core/framework/data_types.h" + +namespace onnxruntime { +namespace featurizers { + +// Forward declarations +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, CatImputerTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, CatImputerTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer); + +Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo + }; + + for (auto& function_table_entry : function_table) { + ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); + } + + return Status::OK(); +} + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/automl_ops/cpu_automl_kernels.h b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.h similarity index 78% rename from onnxruntime/automl_ops/cpu_automl_kernels.h rename to onnxruntime/featurizers_ops/cpu_featurizers_kernels.h index 1f08cdf127..69454e4b60 100644 --- a/onnxruntime/automl_ops/cpu_automl_kernels.h +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.h @@ -7,7 +7,9 @@ #include "core/framework/kernel_registry.h" namespace onnxruntime { -namespace automl { +namespace featurizers { + Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry); -} // namespace automl -} // namespace onnxruntime + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc new file mode 100644 index 0000000000..09080f034b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/CatImputerFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(CategoryImputer, Float_values) { + + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of 1.5 + test.AddInput("State", {4}, {0, 0, 192, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, std::nanf("1"), std::nanf("1"), 2, std::nanf("1")}); + + // Expected output. + test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(CategoryImputer, Double_values) { + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of 1.5 + test.AddInput("State", {8}, {0, 0, 0, 0, 0, 0, 248, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, std::nan("1"), std::nan("1"), 2, std::nan("1")}); + + // Expected output. + test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(CategoryImputer, String_values) { + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of "one" + test.AddInput("State", {7}, {3, 0, 0, 0, 111, 110, 101}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {"ONE", "", "FIVE", "", "NINE"}); + + // Expected output. + test.AddOutput("Output", {5}, {"ONE", "one", "FIVE", "one", "NINE"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/automl_ops/datetimetransformer_test.cc b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc similarity index 97% rename from onnxruntime/test/automl_ops/datetimetransformer_test.cc rename to onnxruntime/test/featurizers_ops/datetimetransformer_test.cc index bb17fefc8c..0887d510b3 100644 --- a/onnxruntime/test/automl_ops/datetimetransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc @@ -15,7 +15,7 @@ namespace test { TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { const time_t date = 217081624; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -77,7 +77,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { const time_t date = 217081625; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -139,7 +139,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 const time_t date1 = 217081625; const time_t date2 = 217081624; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -226,7 +226,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 TEST(DateTimeTransformer, Future_2025_June_30) { const time_t date = 1751241600; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); diff --git a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc b/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc new file mode 100644 index 0000000000..9a8383fd27 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/MaxAbsScalarFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(MaxAbsScaler, Int8_values) { + + OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to Version 1 and a + // scale of 0 + test.AddInput("State", {4}, {0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("X", {5}, {-4,3,0,2,-1}); + + // Expected output. + test.AddOutput("ScaledValues", {5}, {-1,.75,0,.5,-.25}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(MaxAbsScaler, Double_values) { + OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to Version 1 and a + // scale of 0 + test.AddInput("State", {8}, {0, 0, 0, 0, 0, 0, 16, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); + + // Expected output. + test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc new file mode 100644 index 0000000000..6f9b9e8c8b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/StringFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(StringTransformer, Integer_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("Output", {5}, {"1", "3", "5", "7", "9"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, Double_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("Output", {5}, {"1.000000", "3.000000", "5.000000", "7.000000", "9.000000"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, Bool_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {true, false, false, false, true}); + + // Expected output. + test.AddOutput("Output", {5}, {"True", "False", "False", "False", "True"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, String_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"}); + + // Expected output. + test.AddOutput("Output", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/opaque_api/test_opaque_api.cc b/onnxruntime/test/opaque_api/test_opaque_api.cc index 3113663b21..414c9e0500 100644 --- a/onnxruntime/test/opaque_api/test_opaque_api.cc +++ b/onnxruntime/test/opaque_api/test_opaque_api.cc @@ -107,7 +107,7 @@ class OpaqueCApiTestKernel final : public OpKernel { ONNX_OPERATOR_KERNEL_EX( OpaqueCApiTestKernel, - kMSAutoMLDomain, + kMSFeaturizersDomain, 1, kCpuExecutionProvider, KernelDefBuilder() @@ -131,7 +131,7 @@ static void RegisterCustomKernel() { // Registry the schema ONNX_TEST_OPERATOR_SCHEMA(OpaqueCApiTestKernel) .SetDoc("Replace all of h chars to _ in the original string contained within experimental type") - .SetDomain(onnxruntime::kMSAutoMLDomain) + .SetDomain(onnxruntime::kMSFeaturizersDomain) .SinceVersion(1) .Input( 0, @@ -153,7 +153,7 @@ static void RegisterCustomKernel() { // Register kernel directly to KernelRegistry // because we can not create custom ops with Opaque types // as input - BuildKernelCreateInfoFn fn = BuildKernelCreateInfo; + BuildKernelCreateInfoFn fn = BuildKernelCreateInfo; auto kernel_registry = CPUExecutionProvider(CPUExecutionProviderInfo()).GetKernelRegistry(); kernel_registry->Register(fn()); } @@ -179,7 +179,7 @@ std::string CreateModel() { outputs.push_back(&output_arg); auto& node = graph.AddNode("OpaqueCApiTestKernel", "OpaqueCApiTestKernel", "Replace all h to underscore", - inputs, outputs, nullptr, onnxruntime::kMSAutoMLDomain); + inputs, outputs, nullptr, onnxruntime::kMSFeaturizersDomain); node.SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); } EXPECT_TRUE(graph.Resolve().IsOK()); diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 10939a0bdb..d395ebf19c 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -130,7 +130,7 @@ Use the individual flags to only run the specified stages. parser.add_argument("--use_dnnl", action='store_true', help="Build with DNNL.") parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") parser.add_argument("--use_gemmlowp", action='store_true', help="Build with gemmlowp for quantized gemm.") - parser.add_argument("--use_automl", action='store_true', help="Build with AutoML support.") + parser.add_argument("--use_featurizers", action='store_true', help="Build with ML Featurizer support.") parser.add_argument("--use_ngraph", action='store_true', help="Build with nGraph.") parser.add_argument("--use_openvino", nargs="?", const="CPU_FP32", choices=["CPU_FP32","GPU_FP32","GPU_FP16","VAD-M_FP16","MYRIAD_FP16","VAD-F_FP32"], help="Build with OpenVINO for specific hardware.") @@ -290,7 +290,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"), "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""), - "-Donnxruntime_USE_AUTOML=" + ("ON" if args.use_automl else "OFF"), + "-Donnxruntime_USE_FEATURIZERS=" + ("ON" if args.use_featurizers else "OFF"), "-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""), "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"), "-Donnxruntime_USE_MIMALLOC=" + ("ON" if args.use_mimalloc else "OFF"), diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 7063df1dd5..0b925ed087 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -40,7 +40,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -101,7 +101,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -174,7 +174,7 @@ jobs: displayName: 'BUILD' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --enable_onnx_tests --parallel' workingDirectory: '$(Build.BinariesDirectory)' - task: CopyFiles@2 diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 2671762fc3..ec23275b98 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -15,7 +15,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: @@ -39,7 +39,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 267cc75c15..01049e06b6 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU' JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_automl --build_wheel --build_java --enable_language_interop_ops"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_featurizers --build_wheel --build_java --enable_language_interop_ops"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml index 72626a0022..e72ad0c377 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml @@ -3,4 +3,4 @@ jobs: parameters: AgentPool : 'Hosted macOS High Sierra' DoNugetPack: 'false' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_automl --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_featurizers --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index cfec712e79..d7e8bfbfda 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -39,7 +39,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index b08d18602e..6aa762839a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -76,7 +76,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index c43d9e674a..2cf05fc127 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -55,7 +55,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 81914ad074..b7ae98e38c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -43,7 +43,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 40b3ee8bef..5e2df3f9d7 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml index 57d016ebd3..9c3cc6307c 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --x86 --use_openmp --build_shared_lib --enable_onnx_tests' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1