From ce7a180f21fb97c531b073f3590cc928e7bf970b Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 17 Dec 2019 22:17:40 -0800 Subject: [PATCH] Import more featurizers with tests (#2685) Advance commit to 4df80d5865a9d4e97f6d0b9304d4316115a04d9e Add generated code for the commit before editing. Import more featurizers. Rename Automl ops domain to mlfeaturizers. Rename conditional compilation macro. Move and rename files getting rid of automl Rename --use_automl build switch to --use_featurizers Rename CMake option accordingly. Rename automl CMake targets. Adjust CI and packaging pipeline switches. Rename namespace automl to featurizers. --- cgmanifest.json | 2 +- cmake/CMakeLists.txt | 6 +- cmake/external/featurizers.cmake | 40 +- cmake/onnxruntime_graph.cmake | 10 +- cmake/onnxruntime_providers.cmake | 27 +- cmake/onnxruntime_unittests.cmake | 10 +- .../onnxruntime/core/framework/op_kernel.h | 4 +- include/onnxruntime/core/graph/constants.h | 2 +- .../core/session/automl_data_containers.h | 2 +- .../automl_ops/cpu/datetime_transformer.cc | 128 ------- onnxruntime/automl_ops/cpu_automl_kernels.cc | 26 -- .../core/graph/automl_ops/automl_defs.cc | 141 ------- .../core/graph/automl_ops/automl_defs.h | 30 -- .../graph/featurizers_ops/featurizers_defs.cc | 356 ++++++++++++++++++ .../graph/featurizers_ops/featurizers_defs.h | 12 + .../providers/cpu/cpu_execution_provider.cc | 8 +- onnxruntime/core/session/environment.cc | 10 +- .../cpu/CatImputerTransformer.cc | 97 +++++ .../cpu/DateTimeTransformer.cc | 128 +++++++ .../cpu/MaxAbsScalarTransformer.cc | 175 +++++++++ .../featurizers_ops/cpu/StringTransformer.cc | 185 +++++++++ .../cpu_featurizers_kernels.cc | 78 ++++ .../cpu_featurizers_kernels.h} | 8 +- .../featurizers_ops/categoryimputer_test.cc | 64 ++++ .../datetimetransformer_test.cc | 8 +- .../maxabsscalerfeaturizer_test.cc | 48 +++ .../featurizers_ops/stringtransformer_test.cc | 75 ++++ .../test/opaque_api/test_opaque_api.cc | 8 +- tools/ci_build/build.py | 4 +- .../azure-pipelines-py-packaging.yml | 6 +- .../c-api-packaging-pipelines.yml | 4 +- .../azure-pipelines/linux-ci-pipeline.yml | 2 +- .../azure-pipelines/mac-ci-pipeline.yml | 2 +- .../nuget/templates/cpu-mklml.yml | 2 +- .../templates/cpu-nocontribops-arm64.yml | 2 +- .../azure-pipelines/nuget/templates/cpu.yml | 2 +- .../azure-pipelines/nuget/templates/gpu.yml | 2 +- .../azure-pipelines/win-ci-pipeline.yml | 2 +- .../azure-pipelines/win-x86-ci-pipeline.yml | 2 +- 39 files changed, 1292 insertions(+), 426 deletions(-) delete mode 100644 onnxruntime/automl_ops/cpu/datetime_transformer.cc delete mode 100644 onnxruntime/automl_ops/cpu_automl_kernels.cc delete mode 100644 onnxruntime/core/graph/automl_ops/automl_defs.cc delete mode 100644 onnxruntime/core/graph/automl_ops/automl_defs.h create mode 100644 onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc create mode 100644 onnxruntime/core/graph/featurizers_ops/featurizers_defs.h create mode 100644 onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc create mode 100644 onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc create mode 100644 onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc create mode 100644 onnxruntime/featurizers_ops/cpu/StringTransformer.cc create mode 100644 onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc rename onnxruntime/{automl_ops/cpu_automl_kernels.h => featurizers_ops/cpu_featurizers_kernels.h} (78%) create mode 100644 onnxruntime/test/featurizers_ops/categoryimputer_test.cc rename onnxruntime/test/{automl_ops => featurizers_ops}/datetimetransformer_test.cc (97%) create mode 100644 onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc create mode 100644 onnxruntime/test/featurizers_ops/stringtransformer_test.cc diff --git a/cgmanifest.json b/cgmanifest.json index 8c01bf408c..9332ee366a 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -450,7 +450,7 @@ { "component": { "git": { - "commitHash": "006df6bff45dac59d378609fe85f6736a901ee93", + "commitHash": "4df80d5865a9d4e97f6d0b9304d4316115a04d9e", "repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git" }, "type": "git" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 59ca5465e1..2a0d1b03d5 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -55,7 +55,7 @@ option(onnxruntime_USE_NNAPI "Build with DNNLibrary for Android NNAPI support" O option(onnxruntime_USE_DNNL "Build with DNNL support" OFF) option(onnxruntime_USE_MKLML "Build DNNL with MKL-ML binary dependency" OFF) option(onnxruntime_USE_GEMMLOWP "Build with gemmlowp for quantized gemm" OFF) -option(onnxruntime_USE_AUTOML "Build AutoML support" ON) +option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" ON) option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF) option(onnxruntime_USE_OPENBLAS "Use openblas" OFF) option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF) @@ -709,8 +709,8 @@ include(onnxruntime_optimizer.cmake) include(onnxruntime_session.cmake) include(onnxruntime_mlas.cmake) -if(onnxruntime_USE_AUTOML) - add_definitions(-DMICROSOFT_AUTOML) +if(onnxruntime_USE_FEATURIZERS) + add_definitions(-DML_FEATURIZERS) # Fetch and build featurizers include(external/featurizers.cmake) endif() diff --git a/cmake/external/featurizers.cmake b/cmake/external/featurizers.cmake index ba23b4fdef..b9e66570cd 100644 --- a/cmake/external/featurizers.cmake +++ b/cmake/external/featurizers.cmake @@ -3,7 +3,7 @@ # This source code should not depend on the onnxruntime and may be built independently set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git") -set(featurizers_TAG "006df6bff45dac59d378609fe85f6736a901ee93") +set(featurizers_TAG "4df80d5865a9d4e97f6d0b9304d4316115a04d9e") set(featurizers_pref FeaturizersLibrary) set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref}) @@ -41,45 +41,19 @@ else() ) endif() -add_library(automl_featurizers STATIC IMPORTED) -add_dependencies(automl_featurizers featurizers_lib) -target_include_directories(automl_featurizers INTERFACE ${featurizers_ROOT}/src) +add_library(onnxruntime_featurizers STATIC IMPORTED) +add_dependencies(onnxruntime_featurizers featurizers_lib) +target_include_directories(onnxruntime_featurizers INTERFACE ${featurizers_ROOT}/src) if(MSVC) - set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION + set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/${CMAKE_BUILD_TYPE}/FeaturizersCode.lib) else() - set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION + set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/libFeaturizersCode.a) endif() if (WIN32) # Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include. - set_target_properties(automl_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) + set_target_properties(onnxruntime_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props) endif() - -# Build this in CentOS -# foreach(_test_name IN ITEMS - # CatImputerFeaturizer_UnitTests - # DateTimeFeaturizer_UnitTests - # HashOneHotVectorizerFeaturizer_UnitTests - # ImputationMarkerFeaturizer_UnitTests - # LabelEncoderFeaturizer_UnitTests - # MaxAbsScalarFeaturizer_UnitTests - # MinMaxScalarFeaturizer_UnitTests - # MissingDummiesFeaturizer_UnitTests - # OneHotEncoderFeaturizer_UnitTests - # RobustScalarFeaturizer_UnitTests - # SampleAddFeaturizer_UnitTest - # StringFeaturizer_UnitTest - # Structs_UnitTest - # TimeSeriesImputerFeaturizer_UnitTest -# ) - # add_executable(${_test_name} ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp) - # add_dependencies(${_test_name} automl_featurizers) - # target_include_directories(${_test_name} PRIVATE ${featurizers_ROOT}/src) - # target_link_libraries(${_test_name} automl_featurizers) - # list(APPEND featurizers_TEST_SRC ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp) -# endforeach() - -# source_group(TREE ${featurizers_ROOT}/src/Featurizers/UnitTests FILES ${featurizers_TEST_SRC}) diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake index edcf18ae04..ae11ad51e0 100644 --- a/cmake/onnxruntime_graph.cmake +++ b/cmake/onnxruntime_graph.cmake @@ -14,12 +14,12 @@ if (onnxruntime_DISABLE_CONTRIB_OPS) ) endif() -if(NOT onnxruntime_USE_AUTOML) - file(GLOB_RECURSE automl_to_remove_graph_src - "${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.h" - "${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.cc" +if(NOT onnxruntime_USE_FEATURIZERS) + file(GLOB_RECURSE featurizers_to_remove_graph_src + "${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.h" + "${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.cc" ) - foreach(I in ${automl_to_remove_graph_src}) + foreach(I in ${featurizers_to_remove_graph_src}) list(REMOVE_ITEM onnxruntime_graph_src ${I}) endforeach() endif() diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 7549e2b5e2..c480a8ddb8 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -25,14 +25,11 @@ file(GLOB_RECURSE onnxruntime_cuda_contrib_ops_cu_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/contrib_ops/cuda/*.cuh" ) -file(GLOB onnxruntime_cpu_automl_cc_srcs CONFIGURE_DEPENDS - "${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.cc" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_types.h" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_types.cc" - "${ONNXRUNTIME_ROOT}/automl_ops/automl_featurizers.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.h" - "${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.cc" +file(GLOB onnxruntime_cpu_featurizers_cc_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.h" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.cc" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.h" + "${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.cc" ) file(GLOB onnxruntime_providers_common_srcs CONFIGURE_DEPENDS @@ -87,18 +84,18 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS) list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_contrib_ops_srcs}) endif() -if (onnxruntime_USE_AUTOML) - source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_automl_cc_srcs}) - list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_automl_cc_srcs}) +if (onnxruntime_USE_FEATURIZERS) + source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_featurizers_cc_srcs}) + list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_featurizers_cc_srcs}) endif() add_library(onnxruntime_providers ${onnxruntime_providers_src}) onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf) -if (onnxruntime_USE_AUTOML) - add_dependencies(onnxruntime_providers automl_featurizers) - onnxruntime_add_include_to_target(onnxruntime_providers automl_featurizers) - target_link_libraries(onnxruntime_providers automl_featurizers) +if (onnxruntime_USE_FEATURIZERS) + add_dependencies(onnxruntime_providers onnxruntime_featurizers) + onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_featurizers) + target_link_libraries(onnxruntime_providers onnxruntime_featurizers) endif() if(HAS_DEPRECATED_COPY) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 115776cffd..44dbd0017c 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -129,10 +129,10 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS) "${TEST_SRC_DIR}/contrib_ops/*.cc") endif() -if(onnxruntime_USE_AUTOML) +if(onnxruntime_USE_FEATURIZERS) list(APPEND onnxruntime_test_providers_src_patterns - "${TEST_SRC_DIR}/automl_ops/*.h" - "${TEST_SRC_DIR}/automl_ops/*.cc") + "${TEST_SRC_DIR}/featurizers_ops/*.h" + "${TEST_SRC_DIR}/featurizers_ops/*.cc") endif() file(GLOB onnxruntime_test_providers_src CONFIGURE_DEPENDS @@ -218,8 +218,8 @@ if(onnxruntime_USE_NNAPI) list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi) endif() -if(onnxruntime_USE_AUTOML) - list(APPEND onnxruntime_test_providers_dependencies automl_featurizers) +if(onnxruntime_USE_FEATURIZERS) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_featurizers) endif() if(onnxruntime_USE_DML) diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h index 323fd19943..dda7578706 100644 --- a/include/onnxruntime/core/framework/op_kernel.h +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -246,10 +246,10 @@ template KernelCreateInfo BuildKernelCreateInfo(); } // namespace contrib -namespace automl { +namespace featurizers { template KernelCreateInfo BuildKernelCreateInfo(); -} // namespace automl +} // namespace featurizers namespace contrib { namespace cuda { diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index 26fc15d3dd..a4df1eb7b9 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -19,7 +19,7 @@ constexpr const char* kOnnxDomainAlias = "ai.onnx"; constexpr const char* kMLDomain = "ai.onnx.ml"; constexpr const char* kMSDomain = "com.microsoft"; constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc"; -constexpr const char* kMSAutoMLDomain = "com.microsoft.automl"; +constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers"; constexpr const char* kMSDmlDomain = "com.microsoft.dml"; constexpr const char* kNGraphDomain = "com.intel.ai"; constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider"; diff --git a/include/onnxruntime/core/session/automl_data_containers.h b/include/onnxruntime/core/session/automl_data_containers.h index e7f20f77b4..249e42a483 100644 --- a/include/onnxruntime/core/session/automl_data_containers.h +++ b/include/onnxruntime/core/session/automl_data_containers.h @@ -11,7 +11,7 @@ extern "C" { #endif // This structure is used to initialize and read -// OrtValue of opaque(com.microsoft.automl,DateTimeFeaturizer_TimePoint) +// OrtValue of opaque(com.microsoft.featurizers,DateTimeFeaturizer_TimePoint) struct DateTimeFeaturizerTimePointData { int32_t year; uint8_t month; diff --git a/onnxruntime/automl_ops/cpu/datetime_transformer.cc b/onnxruntime/automl_ops/cpu/datetime_transformer.cc deleted file mode 100644 index db3fe6ab20..0000000000 --- a/onnxruntime/automl_ops/cpu/datetime_transformer.cc +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/common/common.h" -#include "core/framework/data_types.h" -#include "core/framework/op_kernel.h" - -#include "Featurizers/DateTimeFeaturizer.h" - -namespace featurizers = Microsoft::Featurizer::Featurizers; - -namespace onnxruntime { -namespace automl { - -class DateTimeTransformer final : public OpKernel { - public: - explicit DateTimeTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { - // Create the transformer - featurizers::DateTimeTransformer transformer( - [ctx]() { - const auto* state_tensor(ctx->Input(0)); - const uint8_t* const state_data(state_tensor->Data()); - - Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); - return featurizers::DateTimeTransformer(archive); - }()); - - // Get the input - const auto* input_tensor(ctx->Input(1)); - const int64_t* input_data(input_tensor->Data()); - - // Prepare the output - Tensor* year_tensor(ctx->Output(0, input_tensor->Shape())); - Tensor* month_tensor(ctx->Output(1, input_tensor->Shape())); - Tensor* day_tensor(ctx->Output(2, input_tensor->Shape())); - Tensor* hour_tensor(ctx->Output(3, input_tensor->Shape())); - Tensor* minute_tensor(ctx->Output(4, input_tensor->Shape())); - Tensor* second_tensor(ctx->Output(5, input_tensor->Shape())); - Tensor* amPm_tensor(ctx->Output(6, input_tensor->Shape())); - Tensor* hour12_tensor(ctx->Output(7, input_tensor->Shape())); - Tensor* dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape())); - Tensor* dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape())); - Tensor* dayOfYear_tensor(ctx->Output(10, input_tensor->Shape())); - Tensor* weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape())); - Tensor* quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape())); - Tensor* halfOfYear_tensor(ctx->Output(13, input_tensor->Shape())); - Tensor* weekIso_tensor(ctx->Output(14, input_tensor->Shape())); - Tensor* yearIso_tensor(ctx->Output(15, input_tensor->Shape())); - Tensor* monthLabel_tensor(ctx->Output(16, input_tensor->Shape())); - Tensor* amPmLabel_tensor(ctx->Output(17, input_tensor->Shape())); - Tensor* dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape())); - Tensor* holidayName_tensor(ctx->Output(19, input_tensor->Shape())); - Tensor* isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape())); - - int32_t* year_data(year_tensor->MutableData()); - uint8_t* month_data(month_tensor->MutableData()); - uint8_t* day_data(day_tensor->MutableData()); - uint8_t* hour_data(hour_tensor->MutableData()); - uint8_t* minute_data(minute_tensor->MutableData()); - uint8_t* second_data(second_tensor->MutableData()); - uint8_t* amPm_data(amPm_tensor->MutableData()); - uint8_t* hour12_data(hour12_tensor->MutableData()); - uint8_t* dayOfWeek_data(dayOfWeek_tensor->MutableData()); - uint8_t* dayOfQuarter_data(dayOfQuarter_tensor->MutableData()); - uint16_t* dayOfYear_data(dayOfYear_tensor->MutableData()); - uint16_t* weekOfMonth_data(weekOfMonth_tensor->MutableData()); - uint8_t* quarterOfYear_data(quarterOfYear_tensor->MutableData()); - uint8_t* halfOfYear_data(halfOfYear_tensor->MutableData()); - uint8_t* weekIso_data(weekIso_tensor->MutableData()); - int32_t* yearIso_data(yearIso_tensor->MutableData()); - std::string* monthLabel_data(monthLabel_tensor->MutableData()); - std::string* amPmLabel_data(amPmLabel_tensor->MutableData()); - std::string* dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData()); - std::string* holidayName_data(holidayName_tensor->MutableData()); - uint8_t* isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData()); - - // Execute - int64_t const length = input_tensor->Shape().Size(); - - for (int64_t i = 0; i < length; ++i) { - auto result(transformer.execute(input_data[i])); - - year_data[i] = std::move(result.year); - month_data[i] = std::move(result.month); - day_data[i] = std::move(result.day); - hour_data[i] = std::move(result.hour); - minute_data[i] = std::move(result.minute); - second_data[i] = std::move(result.second); - amPm_data[i] = std::move(result.amPm); - hour12_data[i] = std::move(result.hour12); - dayOfWeek_data[i] = std::move(result.dayOfWeek); - dayOfQuarter_data[i] = std::move(result.dayOfQuarter); - dayOfYear_data[i] = std::move(result.dayOfYear); - weekOfMonth_data[i] = std::move(result.weekOfMonth); - quarterOfYear_data[i] = std::move(result.quarterOfYear); - halfOfYear_data[i] = std::move(result.halfOfYear); - weekIso_data[i] = std::move(result.weekIso); - yearIso_data[i] = std::move(result.yearIso); - monthLabel_data[i] = std::move(result.monthLabel); - amPmLabel_data[i] = std::move(result.amPmLabel); - dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel); - holidayName_data[i] = std::move(result.holidayName); - isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff); - } - - return Status::OK(); - } -}; - -ONNX_OPERATOR_KERNEL_EX( - DateTimeTransformer, - kMSAutoMLDomain, - 1, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT0", DataTypeImpl::GetTensorType()) - .TypeConstraint("InputT1", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT0", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT1", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT2", DataTypeImpl::GetTensorType()) - .TypeConstraint("OutputT3", DataTypeImpl::GetTensorType()), - DateTimeTransformer); - -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/automl_ops/cpu_automl_kernels.cc b/onnxruntime/automl_ops/cpu_automl_kernels.cc deleted file mode 100644 index e2a0e41ab9..0000000000 --- a/onnxruntime/automl_ops/cpu_automl_kernels.cc +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "automl_ops/cpu_automl_kernels.h" -#include "core/graph/constants.h" -#include "core/framework/data_types.h" - -namespace onnxruntime { -namespace automl { - -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSAutoMLDomain, 1, DateTimeTransformer); - -Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) { - static const BuildKernelCreateInfoFn function_table[] = { - // add more kernels here - BuildKernelCreateInfo - }; - - for (auto& function_table_entry : function_table) { - ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); - } - return Status::OK(); -} - -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/automl_ops/automl_defs.cc b/onnxruntime/core/graph/automl_ops/automl_defs.cc deleted file mode 100644 index 38d8708b37..0000000000 --- a/onnxruntime/core/graph/automl_ops/automl_defs.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/graph/constants.h" -#include "core/graph/automl_ops/automl_defs.h" -#include "core/graph/op.h" -#include "onnx/defs/schema.h" -#include "onnx/defs/shape_inference.h" - -namespace onnxruntime { -namespace automl { -using ONNX_NAMESPACE::AttributeProto; -using ONNX_NAMESPACE::OpSchema; -using ONNX_NAMESPACE::OPTIONAL; - -static const char* DateTimeTransformer_ver1_doc = R"DOC( - Extracts various datetime-related values from a UTC time_point. - - C++-style pseudo signature: - TimePoint execute(std::chron::system_clock::time_point const &value); - - Examples: - Given a time_point 'value' representing "November 17, 1976 12:27:04PM": - - "November 17, 1976 12:27:04PM" => { - "year": 1976, - "month": 11, - "day": 17, - "hour": 12, - "minute": 27, - "second": 04, - "amPm": 2, // PM - "hour12": 12, - "dayOfWeek": 3, // Wednesday - "dayOfQuarter": 48, - "dayOfYear": 321, - "weekOfMonth": 2, - "quarterOfYear": 4, - "halfOfYear": 2, - "weekIso": 47, - "yearIso": 1976, - "monthLabel": "November", - "amPmLabel": "pm", - "dayOfWeekLabel": "Wednesday", - "holidayName": "", - "isPaidTimeOff": 0 - } - )DOC"; - -void RegisterAutoMLSchemas() { - MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer) - .SinceVersion(1) - .SetDomain(kMSAutoMLDomain) - .SetDoc(DateTimeTransformer_ver1_doc) - .Input( - 0, - "State", - "State generated during training that is used for prediction", - "InputT0") - .Input( - 1, - "Input", - "The input represents a number of seconds passed since the epoch, suitable to properly construct" - "an instance of std::chrono::system_clock::time_point", - "InputT1") - .Output(0, "year", "No information available", "OutputT0") - .Output(1, "month", "No information available", "OutputT1") - .Output(2, "day", "No information available", "OutputT1") - .Output(3, "hour", "No information available", "OutputT1") - .Output(4, "minute", "No information available", "OutputT1") - .Output(5, "second", "No information available", "OutputT1") - .Output(6, "amPm", "No information available", "OutputT1") - .Output(7, "hour12", "No information available", "OutputT1") - .Output(8, "dayOfWeek", "No information available", "OutputT1") - .Output(9, "dayOfQuarter", "No information available", "OutputT1") - .Output(10, "dayOfYear", "No information available", "OutputT2") - .Output(11, "weekOfMonth", "No information available", "OutputT2") - .Output(12, "quarterOfYear", "No information available", "OutputT1") - .Output(13, "halfOfYear", "No information available", "OutputT1") - .Output(14, "weekIso", "No information available", "OutputT1") - .Output(15, "yearIso", "No information available", "OutputT0") - .Output(16, "monthLabel", "No information available", "OutputT3") - .Output(17, "amPmLabel", "No information available", "OutputT3") - .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") - .Output(19, "holidayName", "No information available", "OutputT3") - .Output(20, "isPaidTimeOff", "No information available", "OutputT1") - .TypeConstraint( - "InputT0", - {"tensor(uint8)"}, - "No information is available") - .TypeConstraint( - "InputT1", - {"tensor(int64)"}, - "No information is available") - .TypeConstraint( - "OutputT0", - {"tensor(int32)"}, - "No information is available") - .TypeConstraint( - "OutputT1", - {"tensor(uint8)"}, - "No information is available") - .TypeConstraint( - "OutputT2", - {"tensor(uint16)"}, - "No information is available") - .TypeConstraint( - "OutputT3", - {"tensor(string)"}, - "No information is available") - .TypeAndShapeInferenceFunction( - [](ONNX_NAMESPACE::InferenceContext& ctx) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { - *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); - } - }); -} -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/automl_ops/automl_defs.h b/onnxruntime/core/graph/automl_ops/automl_defs.h deleted file mode 100644 index b1a37366c3..0000000000 --- a/onnxruntime/core/graph/automl_ops/automl_defs.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/graph/onnx_protobuf.h" - -namespace onnxruntime { -namespace automl { -#define MS_AUTOML_OPERATOR_SCHEMA(name) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \ - static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ - op_schema_register_once##name##Counter) ONNX_UNUSED = \ - ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__) - -#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) \ - MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) -#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \ - static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ - op_schema_register_once##name##Counter) ONNX_UNUSED = \ - schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)) - -void RegisterAutoMLSchemas(); -} // namespace automl -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc new file mode 100644 index 0000000000..7c820dc6c2 --- /dev/null +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/constants.h" +#include "core/graph/featurizers_ops/featurizers_defs.h" +#include "core/graph/op.h" + +#include "onnx/defs/schema.h" +#include "onnx/defs/shape_inference.h" + +#define MS_AUTOML_OPERATOR_SCHEMA(name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name) +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) + +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__) + +#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func) +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) + +#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \ + static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \ + op_schema_register_once##name##Counter) ONNX_UNUSED = \ + schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)) + +namespace onnxruntime { +namespace featurizers { + +using ONNX_NAMESPACE::AttributeProto; +using ONNX_NAMESPACE::OpSchema; +using ONNX_NAMESPACE::OPTIONAL; + +// Forward declarations +static void RegisterCatImputerFeaturizerVer1(); +static void RegisterDateTimeFeaturizerVer1(); +static void RegisterMaxAbsScalarFeaturizerVer1(); +static void RegisterStringFeaturizerVer1(); + +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +void RegisterAutoMLSchemas() { + RegisterCatImputerFeaturizerVer1(); + RegisterDateTimeFeaturizerVer1(); + RegisterMaxAbsScalarFeaturizerVer1(); + RegisterStringFeaturizerVer1(); +} + +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +// ---------------------------------------------------------------------- +void RegisterCatImputerFeaturizerVer1() { + static const char * doc = R"DOC( + Imputes (populates) values with the mode (most common value) encountered during + training. This featurizer supports float and double for most (if not all) frameworks + due to the existance of NaN in those types. Other types require 'optional' support + within the host frameworks and programming languages. + + C++-style pseudo signature: + std::float_t execute(std::float_t const &value); + std::double_t execute(std::double_t const &value); + template T execute(std::optional const &value); + + Examples (where 55.5 is the mode value): + execute(1.0) -> 1.0 + execute(NaN) -> 55.5 + execute(2.0) -> 2.0 + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(CatImputerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "T" + ) + .Output( + 0, + "Output", + "No information is available", + "T" + ) + .TypeConstraint( + "T", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 1, 0); + if (!hasNInputShapes(ctx, 1)) { + return; + } + propagateShapeFromInputToOutput(ctx, 1, 0); + } + ) + ; +} + +void RegisterDateTimeFeaturizerVer1() { + static const char * doc = R"DOC( + Extracts various datetime-related values from a UTC time_point. + + C++-style pseudo signature: + TimePoint execute(std::chron::system_clock::time_point const &value); + + Examples: + Given a time_point 'value' representing "November 17, 1976 12:27:04PM": + + "November 17, 1976 12:27:04PM" => { + "year": 1976, + "month": 11, + "day": 17, + "hour": 12, + "minute": 27, + "second": 04, + "amPm": 2, // PM + "hour12": 12, + "dayOfWeek": 3, // Wednesday + "dayOfQuarter": 48, + "dayOfYear": 321, + "weekOfMonth": 2, + "quarterOfYear": 4, + "halfOfYear": 2, + "weekIso": 47, + "yearIso": 1976, + "monthLabel": "November", + "amPmLabel": "pm", + "dayOfWeekLabel": "Wednesday", + "holidayName": "", + "isPaidTimeOff": 0 + } + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "tensor(int64)" + ) + .Output(0, "year", "No information available", "OutputT0") + .Output(1, "month", "No information available", "OutputT1") + .Output(2, "day", "No information available", "OutputT1") + .Output(3, "hour", "No information available", "OutputT1") + .Output(4, "minute", "No information available", "OutputT1") + .Output(5, "second", "No information available", "OutputT1") + .Output(6, "amPm", "No information available", "OutputT1") + .Output(7, "hour12", "No information available", "OutputT1") + .Output(8, "dayOfWeek", "No information available", "OutputT1") + .Output(9, "dayOfQuarter", "No information available", "OutputT1") + .Output(10, "dayOfYear", "No information available", "OutputT2") + .Output(11, "weekOfMonth", "No information available", "OutputT2") + .Output(12, "quarterOfYear", "No information available", "OutputT1") + .Output(13, "halfOfYear", "No information available", "OutputT1") + .Output(14, "weekIso", "No information available", "OutputT1") + .Output(15, "yearIso", "No information available", "OutputT0") + .Output(16, "monthLabel", "No information available", "OutputT3") + .Output(17, "amPmLabel", "No information available", "OutputT3") + .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") + .Output(19, "holidayName", "No information available", "OutputT3") + .Output(20, "isPaidTimeOff", "No information available", "OutputT1") + .TypeConstraint( + "OutputT0", + {"tensor(int32)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT1", + {"tensor(uint8)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT2", + {"tensor(uint16)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT3", + {"tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext &ctx) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); + ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); + ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); + ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); + ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); + ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + + for(size_t i = 0; i < ctx.getNumOutputs(); ++i) { + *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + } + ) + ; +} + +void RegisterMaxAbsScalarFeaturizerVer1() { + static const char * doc = R"DOC( + Scales input based on the maximum absolute value of all data encountered during training. + + C++-style pseudo signature: + std::float_t execute(std::uint16_t value); + std::double_t execute(std::uint32_t value); + + Examples: + Given a training set of [1.0, -2.0, 3.0, -4.0], where 4.0 is the absolute value of the + maximum value encountered... + + execute(1.0) -> 1.0 / 4.0 + execute(-4.0) -> -4.0 / 4.0 + execute(100.0) -> 100 / 4.0 + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(MaxAbsScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "InputT" + ) + .Output( + 0, + "Output", + "No information is available", + "OutputT" + ) + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, + "No information is available" + ) + .TypeConstraint( + "OutputT", + {"tensor(float)", "tensor(double)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + auto input_elem_type = ctx.getInputType(1)->tensor_type().elem_type(); + if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE); + } + + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + ) + ; +} + +void RegisterStringFeaturizerVer1() { + static const char * doc = R"DOC( + Converts the input into a string representation based on the input's type. + + C++-style pseudo signature: + template std::string execute(T const &value); + + Examples: + execute(1) -> "1" + execute(3.14) -> "3.14" + )DOC"; + + MS_AUTOML_OPERATOR_SCHEMA(StringTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "tensor(uint8)" + ) + .Input( + 1, + "Input", + "No information is available", + "InputT" + ) + .Output( + 0, + "Output", + "No information is available", + "tensor(string)" + ) + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available" + ) + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 0); + + *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + } + ) + ; +} + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h new file mode 100644 index 0000000000..d1323c0b3c --- /dev/null +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +namespace onnxruntime { +namespace featurizers { + +void RegisterAutoMLSchemas(void); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index d56ab62fcb..77e7370dfa 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -9,8 +9,8 @@ #include "contrib_ops/cpu_contrib_kernels.h" #endif -#ifdef MICROSOFT_AUTOML -#include "automl_ops/cpu_automl_kernels.h" +#ifdef ML_FEATURIZERS +#include "featurizers_ops/cpu_featurizers_kernels.h" #endif #include "core/framework/compute_capability.h" @@ -1270,8 +1270,8 @@ static Status RegisterCPUKernels(KernelRegistry& kernel_registry) { #ifndef DISABLE_CONTRIB_OPS ORT_RETURN_IF_ERROR(::onnxruntime::contrib::RegisterCpuContribKernels(kernel_registry)); #endif -#ifdef MICROSOFT_AUTOML - ORT_RETURN_IF_ERROR(::onnxruntime::automl::RegisterCpuAutoMLKernels(kernel_registry)); +#ifdef ML_FEATURIZERS + ORT_RETURN_IF_ERROR(::onnxruntime::featurizers::RegisterCpuAutoMLKernels(kernel_registry)); #endif return Status::OK(); } diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index da0281e643..d133b9d997 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -10,8 +10,8 @@ #ifndef DISABLE_CONTRIB_OPS #include "core/graph/contrib_ops/contrib_defs.h" #endif -#ifdef MICROSOFT_AUTOML -#include "core/graph/automl_ops/automl_defs.h" +#ifdef ML_FEATURIZERS +#include "core/graph/featurizers_ops/featurizers_defs.h" #endif #ifdef USE_DML #include "core/graph/dml_ops/dml_defs.h" @@ -45,7 +45,7 @@ Status Environment::Initialize() { std::call_once(schemaRegistrationOnceFlag, []() { ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 1); ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSNchwcDomain, 1, 1); - ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSAutoMLDomain, 1, 1); + ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSFeaturizersDomain, 1, 1); #ifdef USE_DML ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDmlDomain, 1, 1); #endif @@ -54,8 +54,8 @@ Status Environment::Initialize() { #ifndef DISABLE_CONTRIB_OPS contrib::RegisterContribSchemas(); #endif -#ifdef MICROSOFT_AUTOML - automl::RegisterAutoMLSchemas(); +#ifdef ML_FEATURIZERS + featurizers::RegisterAutoMLSchemas(); #endif #ifdef USE_DML dml::RegisterDmlSchemas(); diff --git a/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc b/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc new file mode 100644 index 0000000000..0d4daa51ad --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/CatImputerFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template +struct OutputTypeMapper {}; +template <> +struct OutputTypeMapper { using type = float_t; }; +template <> +struct OutputTypeMapper { using type = double_t; }; +template <> +struct OutputTypeMapper { using type = std::string; }; + +inline float_t const& PreprocessOptional(float_t const& value) { return value; } +inline double_t const& PreprocessOptional(double_t const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +class CatImputerTransformer final : public OpKernel { + public: + explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + // Create the transformer + feat::CatImputerTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::CatImputerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const T* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type* output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + CatImputerTransformer, + kMSFeaturizersDomain, + 1, + string, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()), + CatImputerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc b/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc new file mode 100644 index 0000000000..013222154c --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/DateTimeFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +class DateTimeTransformer final : public OpKernel { +public: + explicit DateTimeTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::DateTimeTransformer transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::DateTimeTransformer(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const std::int64_t * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * year_tensor(ctx->Output(0, input_tensor->Shape())); + Tensor * month_tensor(ctx->Output(1, input_tensor->Shape())); + Tensor * day_tensor(ctx->Output(2, input_tensor->Shape())); + Tensor * hour_tensor(ctx->Output(3, input_tensor->Shape())); + Tensor * minute_tensor(ctx->Output(4, input_tensor->Shape())); + Tensor * second_tensor(ctx->Output(5, input_tensor->Shape())); + Tensor * amPm_tensor(ctx->Output(6, input_tensor->Shape())); + Tensor * hour12_tensor(ctx->Output(7, input_tensor->Shape())); + Tensor * dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape())); + Tensor * dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape())); + Tensor * dayOfYear_tensor(ctx->Output(10, input_tensor->Shape())); + Tensor * weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape())); + Tensor * quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape())); + Tensor * halfOfYear_tensor(ctx->Output(13, input_tensor->Shape())); + Tensor * weekIso_tensor(ctx->Output(14, input_tensor->Shape())); + Tensor * yearIso_tensor(ctx->Output(15, input_tensor->Shape())); + Tensor * monthLabel_tensor(ctx->Output(16, input_tensor->Shape())); + Tensor * amPmLabel_tensor(ctx->Output(17, input_tensor->Shape())); + Tensor * dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape())); + Tensor * holidayName_tensor(ctx->Output(19, input_tensor->Shape())); + Tensor * isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape())); + + int32_t * year_data(year_tensor->MutableData()); + uint8_t * month_data(month_tensor->MutableData()); + uint8_t * day_data(day_tensor->MutableData()); + uint8_t * hour_data(hour_tensor->MutableData()); + uint8_t * minute_data(minute_tensor->MutableData()); + uint8_t * second_data(second_tensor->MutableData()); + uint8_t * amPm_data(amPm_tensor->MutableData()); + uint8_t * hour12_data(hour12_tensor->MutableData()); + uint8_t * dayOfWeek_data(dayOfWeek_tensor->MutableData()); + uint8_t * dayOfQuarter_data(dayOfQuarter_tensor->MutableData()); + uint16_t * dayOfYear_data(dayOfYear_tensor->MutableData()); + uint16_t * weekOfMonth_data(weekOfMonth_tensor->MutableData()); + uint8_t * quarterOfYear_data(quarterOfYear_tensor->MutableData()); + uint8_t * halfOfYear_data(halfOfYear_tensor->MutableData()); + uint8_t * weekIso_data(weekIso_tensor->MutableData()); + int32_t * yearIso_data(yearIso_tensor->MutableData()); + std::string * monthLabel_data(monthLabel_tensor->MutableData()); + std::string * amPmLabel_data(amPmLabel_tensor->MutableData()); + std::string * dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData()); + std::string * holidayName_data(holidayName_tensor->MutableData()); + uint8_t * isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + auto result(transformer.execute(input_data[i])); + + year_data[i] = std::move(result.year); + month_data[i] = std::move(result.month); + day_data[i] = std::move(result.day); + hour_data[i] = std::move(result.hour); + minute_data[i] = std::move(result.minute); + second_data[i] = std::move(result.second); + amPm_data[i] = std::move(result.amPm); + hour12_data[i] = std::move(result.hour12); + dayOfWeek_data[i] = std::move(result.dayOfWeek); + dayOfQuarter_data[i] = std::move(result.dayOfQuarter); + dayOfYear_data[i] = std::move(result.dayOfYear); + weekOfMonth_data[i] = std::move(result.weekOfMonth); + quarterOfYear_data[i] = std::move(result.quarterOfYear); + halfOfYear_data[i] = std::move(result.halfOfYear); + weekIso_data[i] = std::move(result.weekIso); + yearIso_data[i] = std::move(result.yearIso); + monthLabel_data[i] = std::move(result.monthLabel); + amPmLabel_data[i] = std::move(result.amPmLabel); + dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel); + holidayName_data[i] = std::move(result.holidayName); + isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + DateTimeTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("OutputT0", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT1", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT2", DataTypeImpl::GetTensorType()) + .TypeConstraint("OutputT3", DataTypeImpl::GetTensorType()), + DateTimeTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc b/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc new file mode 100644 index 0000000000..3ad2ad5b8b --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MaxAbsScalarFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template struct OutputTypeMapper {}; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = float_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; +template <> struct OutputTypeMapper { using type = double_t; }; + +template +class MaxAbsScalarTransformer final : public OpKernel { +public: + explicit MaxAbsScalarTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::MaxAbsScalarTransformer::type> transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::MaxAbsScalarTransformer::type>(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type * output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + int64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + uint64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + MaxAbsScalarTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + MaxAbsScalarTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/StringTransformer.cc b/onnxruntime/featurizers_ops/cpu/StringTransformer.cc new file mode 100644 index 0000000000..48a5ca424d --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/StringTransformer.cc @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/StringFeaturizer.h" + +namespace feat = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace featurizers { + +template +class StringTransformer final : public OpKernel { +public: + explicit StringTransformer(const OpKernelInfo &info) : OpKernel(info) { + } + + Status Compute(OpKernelContext *ctx) const override { + // Create the transformer + feat::StringTransformer transformer( + [ctx](void) { + const auto * state_tensor(ctx->Input(0)); + const uint8_t * const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return feat::StringTransformer(archive); + }() + ); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT * input_data(input_tensor->Data()); + + // Prepare the output + Tensor * output_tensor(ctx->Output(0, input_tensor->Shape())); + std::string * output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for(int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + + return Status::OK(); + } +}; + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + int64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint8_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint16_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint32_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + uint64_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + float_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + double_t, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + bool, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +ONNX_OPERATOR_TYPED_KERNEL_EX( + StringTransformer, + kMSFeaturizersDomain, + 1, + string, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + StringTransformer +); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc new file mode 100644 index 0000000000..0e11490b7d --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "featurizers_ops/cpu_featurizers_kernels.h" + +#include "core/graph/constants.h" +#include "core/framework/data_types.h" + +namespace onnxruntime { +namespace featurizers { + +// Forward declarations +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, CatImputerTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, CatImputerTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, MaxAbsScalarTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer); + +Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) { + static const BuildKernelCreateInfoFn function_table[] = { + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo + }; + + for (auto& function_table_entry : function_table) { + ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); + } + + return Status::OK(); +} + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/automl_ops/cpu_automl_kernels.h b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.h similarity index 78% rename from onnxruntime/automl_ops/cpu_automl_kernels.h rename to onnxruntime/featurizers_ops/cpu_featurizers_kernels.h index 1f08cdf127..69454e4b60 100644 --- a/onnxruntime/automl_ops/cpu_automl_kernels.h +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.h @@ -7,7 +7,9 @@ #include "core/framework/kernel_registry.h" namespace onnxruntime { -namespace automl { +namespace featurizers { + Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry); -} // namespace automl -} // namespace onnxruntime + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc new file mode 100644 index 0000000000..09080f034b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/CatImputerFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(CategoryImputer, Float_values) { + + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of 1.5 + test.AddInput("State", {4}, {0, 0, 192, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, std::nanf("1"), std::nanf("1"), 2, std::nanf("1")}); + + // Expected output. + test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(CategoryImputer, Double_values) { + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of 1.5 + test.AddInput("State", {8}, {0, 0, 0, 0, 0, 0, 248, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, std::nan("1"), std::nan("1"), 2, std::nan("1")}); + + // Expected output. + test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(CategoryImputer, String_values) { + OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to a + // most frequent value during training of "one" + test.AddInput("State", {7}, {3, 0, 0, 0, 111, 110, 101}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {"ONE", "", "FIVE", "", "NINE"}); + + // Expected output. + test.AddOutput("Output", {5}, {"ONE", "one", "FIVE", "one", "NINE"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/automl_ops/datetimetransformer_test.cc b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc similarity index 97% rename from onnxruntime/test/automl_ops/datetimetransformer_test.cc rename to onnxruntime/test/featurizers_ops/datetimetransformer_test.cc index bb17fefc8c..0887d510b3 100644 --- a/onnxruntime/test/automl_ops/datetimetransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc @@ -15,7 +15,7 @@ namespace test { TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { const time_t date = 217081624; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -77,7 +77,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { const time_t date = 217081625; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -139,7 +139,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 const time_t date1 = 217081625; const time_t date2 = 217081624; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); @@ -226,7 +226,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 TEST(DateTimeTransformer, Future_2025_June_30) { const time_t date = 1751241600; - OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain); + OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); // Add state input test.AddInput("State", {4}, {0, 0, 0, 0}); diff --git a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc b/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc new file mode 100644 index 0000000000..9a8383fd27 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/MaxAbsScalarFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(MaxAbsScaler, Int8_values) { + + OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to Version 1 and a + // scale of 0 + test.AddInput("State", {4}, {0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("X", {5}, {-4,3,0,2,-1}); + + // Expected output. + test.AddOutput("ScaledValues", {5}, {-1,.75,0,.5,-.25}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(MaxAbsScaler, Double_values) { + OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State from when the transformer was trained. Corresponds to Version 1 and a + // scale of 0 + test.AddInput("State", {8}, {0, 0, 0, 0, 0, 0, 16, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); + + // Expected output. + test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc new file mode 100644 index 0000000000..6f9b9e8c8b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/StringFeaturizer.h" + +namespace dft = Microsoft::Featurizer::Featurizers; + +namespace onnxruntime { +namespace test { + +TEST(StringTransformer, Integer_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("Output", {5}, {"1", "3", "5", "7", "9"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, Double_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("Output", {5}, {"1.000000", "3.000000", "5.000000", "7.000000", "9.000000"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, Bool_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {true, false, false, false, true}); + + // Expected output. + test.AddOutput("Output", {5}, {"True", "False", "False", "False", "True"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(StringTransformer, String_values) { + OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // State input is needed, but no actual state is required. + test.AddInput("State", {0}, {}); + + // We are adding a scalar Tensor in this instance + test.AddInput("Input", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"}); + + // Expected output. + test.AddOutput("Output", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/opaque_api/test_opaque_api.cc b/onnxruntime/test/opaque_api/test_opaque_api.cc index 3113663b21..414c9e0500 100644 --- a/onnxruntime/test/opaque_api/test_opaque_api.cc +++ b/onnxruntime/test/opaque_api/test_opaque_api.cc @@ -107,7 +107,7 @@ class OpaqueCApiTestKernel final : public OpKernel { ONNX_OPERATOR_KERNEL_EX( OpaqueCApiTestKernel, - kMSAutoMLDomain, + kMSFeaturizersDomain, 1, kCpuExecutionProvider, KernelDefBuilder() @@ -131,7 +131,7 @@ static void RegisterCustomKernel() { // Registry the schema ONNX_TEST_OPERATOR_SCHEMA(OpaqueCApiTestKernel) .SetDoc("Replace all of h chars to _ in the original string contained within experimental type") - .SetDomain(onnxruntime::kMSAutoMLDomain) + .SetDomain(onnxruntime::kMSFeaturizersDomain) .SinceVersion(1) .Input( 0, @@ -153,7 +153,7 @@ static void RegisterCustomKernel() { // Register kernel directly to KernelRegistry // because we can not create custom ops with Opaque types // as input - BuildKernelCreateInfoFn fn = BuildKernelCreateInfo; + BuildKernelCreateInfoFn fn = BuildKernelCreateInfo; auto kernel_registry = CPUExecutionProvider(CPUExecutionProviderInfo()).GetKernelRegistry(); kernel_registry->Register(fn()); } @@ -179,7 +179,7 @@ std::string CreateModel() { outputs.push_back(&output_arg); auto& node = graph.AddNode("OpaqueCApiTestKernel", "OpaqueCApiTestKernel", "Replace all h to underscore", - inputs, outputs, nullptr, onnxruntime::kMSAutoMLDomain); + inputs, outputs, nullptr, onnxruntime::kMSFeaturizersDomain); node.SetExecutionProviderType(onnxruntime::kCpuExecutionProvider); } EXPECT_TRUE(graph.Resolve().IsOK()); diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 10939a0bdb..d395ebf19c 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -130,7 +130,7 @@ Use the individual flags to only run the specified stages. parser.add_argument("--use_dnnl", action='store_true', help="Build with DNNL.") parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") parser.add_argument("--use_gemmlowp", action='store_true', help="Build with gemmlowp for quantized gemm.") - parser.add_argument("--use_automl", action='store_true', help="Build with AutoML support.") + parser.add_argument("--use_featurizers", action='store_true', help="Build with ML Featurizer support.") parser.add_argument("--use_ngraph", action='store_true', help="Build with nGraph.") parser.add_argument("--use_openvino", nargs="?", const="CPU_FP32", choices=["CPU_FP32","GPU_FP32","GPU_FP16","VAD-M_FP16","MYRIAD_FP16","VAD-F_FP32"], help="Build with OpenVINO for specific hardware.") @@ -290,7 +290,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"), "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""), - "-Donnxruntime_USE_AUTOML=" + ("ON" if args.use_automl else "OFF"), + "-Donnxruntime_USE_FEATURIZERS=" + ("ON" if args.use_featurizers else "OFF"), "-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""), "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"), "-Donnxruntime_USE_MIMALLOC=" + ("ON" if args.use_mimalloc else "OFF"), diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 7063df1dd5..0b925ed087 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -40,7 +40,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -101,7 +101,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -174,7 +174,7 @@ jobs: displayName: 'BUILD' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --enable_onnx_tests --parallel' workingDirectory: '$(Build.BinariesDirectory)' - task: CopyFiles@2 diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 2671762fc3..ec23275b98 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -15,7 +15,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: @@ -39,7 +39,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 267cc75c15..01049e06b6 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU' JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_automl --build_wheel --build_java --enable_language_interop_ops"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_featurizers --build_wheel --build_java --enable_language_interop_ops"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml index 72626a0022..e72ad0c377 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml @@ -3,4 +3,4 @@ jobs: parameters: AgentPool : 'Hosted macOS High Sierra' DoNugetPack: 'false' - BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_automl --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_featurizers --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index cfec712e79..d7e8bfbfda 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -39,7 +39,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index b08d18602e..6aa762839a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -76,7 +76,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index c43d9e674a..2cf05fc127 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -55,7 +55,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 81914ad074..b7ae98e38c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -43,7 +43,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 40b3ee8bef..5e2df3f9d7 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 diff --git a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml index 57d016ebd3..9c3cc6307c 100644 --- a/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --x86 --use_openmp --build_shared_lib --enable_onnx_tests' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1