saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Import more featurizers with tests (#2685)

Browse source 
Advance commit to 4df80d5865a9d4e97f6d0b9304d4316115a04d9e
  Add generated code for the commit before editing.
  Import more featurizers.
  Rename Automl ops domain to mlfeaturizers.
  Rename conditional compilation macro.
  Move and rename files getting rid of automl
  Rename --use_automl build switch to --use_featurizers
  Rename CMake option accordingly. Rename automl CMake targets.
  Adjust CI and packaging pipeline switches.
  Rename namespace automl to featurizers.
This commit is contained in:
Dmitri Smirnov 2019-12-17 22:17:40 -08:00 committed by GitHub
parent c767e264c5
commit ce7a180f21
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 1292 additions and 426 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cgmanifest.json
View file

@ -450,7 +450,7 @@
{
"component": {
"git": {
"commitHash": "006df6bff45dac59d378609fe85f6736a901ee93",
"commitHash": "4df80d5865a9d4e97f6d0b9304d4316115a04d9e",
"repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git"
},
"type": "git"

6
cmake/CMakeLists.txt
View file

@ -55,7 +55,7 @@ option(onnxruntime_USE_NNAPI "Build with DNNLibrary for Android NNAPI support" O
option(onnxruntime_USE_DNNL "Build with DNNL support" OFF)
option(onnxruntime_USE_MKLML "Build DNNL with MKL-ML binary dependency" OFF)
option(onnxruntime_USE_GEMMLOWP "Build with gemmlowp for quantized gemm" OFF)
option(onnxruntime_USE_AUTOML "Build AutoML support" ON)
option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" ON)
option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF)
option(onnxruntime_USE_OPENBLAS "Use openblas" OFF)
option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF)
@ -709,8 +709,8 @@ include(onnxruntime_optimizer.cmake)
include(onnxruntime_session.cmake)
include(onnxruntime_mlas.cmake)
if(onnxruntime_USE_AUTOML)
add_definitions(-DMICROSOFT_AUTOML)
if(onnxruntime_USE_FEATURIZERS)
add_definitions(-DML_FEATURIZERS)
# Fetch and build featurizers
include(external/featurizers.cmake)
endif()

40
cmake/external/featurizers.cmake vendored
View file

@ -3,7 +3,7 @@
# This source code should not depend on the onnxruntime and may be built independently
set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git")
set(featurizers_TAG "006df6bff45dac59d378609fe85f6736a901ee93")
set(featurizers_TAG "4df80d5865a9d4e97f6d0b9304d4316115a04d9e")
set(featurizers_pref FeaturizersLibrary)
set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref})
@ -41,45 +41,19 @@ else()
)
endif()
add_library(automl_featurizers STATIC IMPORTED)
add_dependencies(automl_featurizers featurizers_lib)
target_include_directories(automl_featurizers INTERFACE ${featurizers_ROOT}/src)
add_library(onnxruntime_featurizers STATIC IMPORTED)
add_dependencies(onnxruntime_featurizers featurizers_lib)
target_include_directories(onnxruntime_featurizers INTERFACE ${featurizers_ROOT}/src)
if(MSVC)
set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION
set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION
${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/${CMAKE_BUILD_TYPE}/FeaturizersCode.lib)
else()
set_property(TARGET automl_featurizers PROPERTY IMPORTED_LOCATION
set_property(TARGET onnxruntime_featurizers PROPERTY IMPORTED_LOCATION
${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}/libFeaturizersCode.a)
endif()
if (WIN32)
# Add Code Analysis properties to enable C++ Core checks. Have to do it via a props file include.
set_target_properties(automl_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props)
set_target_properties(onnxruntime_featurizers PROPERTIES VS_USER_PROPS ${PROJECT_SOURCE_DIR}/ConfigureVisualStudioCodeAnalysis.props)
endif()
# Build this in CentOS
# foreach(_test_name IN ITEMS
# CatImputerFeaturizer_UnitTests
# DateTimeFeaturizer_UnitTests
# HashOneHotVectorizerFeaturizer_UnitTests
# ImputationMarkerFeaturizer_UnitTests
# LabelEncoderFeaturizer_UnitTests
# MaxAbsScalarFeaturizer_UnitTests
# MinMaxScalarFeaturizer_UnitTests
# MissingDummiesFeaturizer_UnitTests
# OneHotEncoderFeaturizer_UnitTests
# RobustScalarFeaturizer_UnitTests
# SampleAddFeaturizer_UnitTest
# StringFeaturizer_UnitTest
# Structs_UnitTest
# TimeSeriesImputerFeaturizer_UnitTest
# )
# add_executable(${_test_name} ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp)
# add_dependencies(${_test_name} automl_featurizers)
# target_include_directories(${_test_name} PRIVATE ${featurizers_ROOT}/src)
# target_link_libraries(${_test_name} automl_featurizers)
# list(APPEND featurizers_TEST_SRC ${featurizers_ROOT}/src/Featurizers/UnitTests/${_test_name}.cpp)
# endforeach()
# source_group(TREE ${featurizers_ROOT}/src/Featurizers/UnitTests FILES ${featurizers_TEST_SRC})

10
cmake/onnxruntime_graph.cmake
View file

@ -14,12 +14,12 @@ if (onnxruntime_DISABLE_CONTRIB_OPS)
)
endif()
if(NOT onnxruntime_USE_AUTOML)
file(GLOB_RECURSE automl_to_remove_graph_src
"${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.h"
"${ONNXRUNTIME_ROOT}/core/graph/automl_ops/*.cc"
if(NOT onnxruntime_USE_FEATURIZERS)
file(GLOB_RECURSE featurizers_to_remove_graph_src
"${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.h"
"${ONNXRUNTIME_ROOT}/core/graph/featurizers_ops/*.cc"
)
foreach(I in ${automl_to_remove_graph_src})
foreach(I in ${featurizers_to_remove_graph_src})
list(REMOVE_ITEM onnxruntime_graph_src ${I})
endforeach()
endif()

27
cmake/onnxruntime_providers.cmake
View file

@ -25,14 +25,11 @@ file(GLOB_RECURSE onnxruntime_cuda_contrib_ops_cu_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/contrib_ops/cuda/*.cuh"
)
file(GLOB onnxruntime_cpu_automl_cc_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.h"
"${ONNXRUNTIME_ROOT}/automl_ops/cpu_automl_kernels.cc"
"${ONNXRUNTIME_ROOT}/automl_ops/automl_types.h"
"${ONNXRUNTIME_ROOT}/automl_ops/automl_types.cc"
"${ONNXRUNTIME_ROOT}/automl_ops/automl_featurizers.h"
"${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.h"
"${ONNXRUNTIME_ROOT}/automl_ops/cpu/*.cc"
file(GLOB onnxruntime_cpu_featurizers_cc_srcs CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.h"
"${ONNXRUNTIME_ROOT}/featurizers_ops/cpu_featurizers_kernels.cc"
"${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.h"
"${ONNXRUNTIME_ROOT}/featurizers_ops/cpu/*.cc"
)
file(GLOB onnxruntime_providers_common_srcs CONFIGURE_DEPENDS
@ -87,18 +84,18 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS)
list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_contrib_ops_srcs})
endif()
if (onnxruntime_USE_AUTOML)
source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_automl_cc_srcs})
list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_automl_cc_srcs})
if (onnxruntime_USE_FEATURIZERS)
source_group(TREE ${ONNXRUNTIME_ROOT}/ FILES ${onnxruntime_cpu_featurizers_cc_srcs})
list(APPEND onnxruntime_providers_src ${onnxruntime_cpu_featurizers_cc_srcs})
endif()
add_library(onnxruntime_providers ${onnxruntime_providers_src})
onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_common onnxruntime_framework onnx onnx_proto protobuf::libprotobuf)
if (onnxruntime_USE_AUTOML)
add_dependencies(onnxruntime_providers automl_featurizers)
onnxruntime_add_include_to_target(onnxruntime_providers automl_featurizers)
target_link_libraries(onnxruntime_providers automl_featurizers)
if (onnxruntime_USE_FEATURIZERS)
add_dependencies(onnxruntime_providers onnxruntime_featurizers)
onnxruntime_add_include_to_target(onnxruntime_providers onnxruntime_featurizers)
target_link_libraries(onnxruntime_providers onnxruntime_featurizers)
endif()
if(HAS_DEPRECATED_COPY)

10
cmake/onnxruntime_unittests.cmake
View file

@ -129,10 +129,10 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS)
"${TEST_SRC_DIR}/contrib_ops/*.cc")
endif()
if(onnxruntime_USE_AUTOML)
if(onnxruntime_USE_FEATURIZERS)
list(APPEND onnxruntime_test_providers_src_patterns
"${TEST_SRC_DIR}/automl_ops/*.h"
"${TEST_SRC_DIR}/automl_ops/*.cc")
"${TEST_SRC_DIR}/featurizers_ops/*.h"
"${TEST_SRC_DIR}/featurizers_ops/*.cc")
endif()
file(GLOB onnxruntime_test_providers_src CONFIGURE_DEPENDS
@ -218,8 +218,8 @@ if(onnxruntime_USE_NNAPI)
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nnapi)
endif()
if(onnxruntime_USE_AUTOML)
list(APPEND onnxruntime_test_providers_dependencies automl_featurizers)
if(onnxruntime_USE_FEATURIZERS)
list(APPEND onnxruntime_test_providers_dependencies onnxruntime_featurizers)
endif()
if(onnxruntime_USE_DML)

4
include/onnxruntime/core/framework/op_kernel.h
View file

@ -246,10 +246,10 @@ template <typename T>
KernelCreateInfo BuildKernelCreateInfo();
} // namespace contrib
namespace automl {
namespace featurizers {
template <typename T>
KernelCreateInfo BuildKernelCreateInfo();
} // namespace automl
} // namespace featurizers
namespace contrib {
namespace cuda {

2
include/onnxruntime/core/graph/constants.h
View file

@ -19,7 +19,7 @@ constexpr const char* kOnnxDomainAlias = "ai.onnx";
constexpr const char* kMLDomain = "ai.onnx.ml";
constexpr const char* kMSDomain = "com.microsoft";
constexpr const char* kMSNchwcDomain = "com.microsoft.nchwc";
constexpr const char* kMSAutoMLDomain = "com.microsoft.automl";
constexpr const char* kMSFeaturizersDomain = "com.microsoft.mlfeaturizers";
constexpr const char* kMSDmlDomain = "com.microsoft.dml";
constexpr const char* kNGraphDomain = "com.intel.ai";
constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider";

2
include/onnxruntime/core/session/automl_data_containers.h
View file

@ -11,7 +11,7 @@
extern "C" {
#endif
// This structure is used to initialize and read
// OrtValue of opaque(com.microsoft.automl,DateTimeFeaturizer_TimePoint)
// OrtValue of opaque(com.microsoft.featurizers,DateTimeFeaturizer_TimePoint)
struct DateTimeFeaturizerTimePointData {
int32_t year;
uint8_t month;

128
onnxruntime/automl_ops/cpu/datetime_transformer.cc
View file

@ -1,128 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/DateTimeFeaturizer.h"
namespace featurizers = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace automl {
class DateTimeTransformer final : public OpKernel {
public:
explicit DateTimeTransformer(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* ctx) const override {
// Create the transformer
featurizers::DateTimeTransformer transformer(
[ctx]() {
const auto* state_tensor(ctx->Input<Tensor>(0));
const uint8_t* const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return featurizers::DateTimeTransformer(archive);
}());
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const int64_t* input_data(input_tensor->Data<int64_t>());
// Prepare the output
Tensor* year_tensor(ctx->Output(0, input_tensor->Shape()));
Tensor* month_tensor(ctx->Output(1, input_tensor->Shape()));
Tensor* day_tensor(ctx->Output(2, input_tensor->Shape()));
Tensor* hour_tensor(ctx->Output(3, input_tensor->Shape()));
Tensor* minute_tensor(ctx->Output(4, input_tensor->Shape()));
Tensor* second_tensor(ctx->Output(5, input_tensor->Shape()));
Tensor* amPm_tensor(ctx->Output(6, input_tensor->Shape()));
Tensor* hour12_tensor(ctx->Output(7, input_tensor->Shape()));
Tensor* dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape()));
Tensor* dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape()));
Tensor* dayOfYear_tensor(ctx->Output(10, input_tensor->Shape()));
Tensor* weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape()));
Tensor* quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape()));
Tensor* halfOfYear_tensor(ctx->Output(13, input_tensor->Shape()));
Tensor* weekIso_tensor(ctx->Output(14, input_tensor->Shape()));
Tensor* yearIso_tensor(ctx->Output(15, input_tensor->Shape()));
Tensor* monthLabel_tensor(ctx->Output(16, input_tensor->Shape()));
Tensor* amPmLabel_tensor(ctx->Output(17, input_tensor->Shape()));
Tensor* dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape()));
Tensor* holidayName_tensor(ctx->Output(19, input_tensor->Shape()));
Tensor* isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape()));
int32_t* year_data(year_tensor->MutableData<int32_t>());
uint8_t* month_data(month_tensor->MutableData<uint8_t>());
uint8_t* day_data(day_tensor->MutableData<uint8_t>());
uint8_t* hour_data(hour_tensor->MutableData<uint8_t>());
uint8_t* minute_data(minute_tensor->MutableData<uint8_t>());
uint8_t* second_data(second_tensor->MutableData<uint8_t>());
uint8_t* amPm_data(amPm_tensor->MutableData<uint8_t>());
uint8_t* hour12_data(hour12_tensor->MutableData<uint8_t>());
uint8_t* dayOfWeek_data(dayOfWeek_tensor->MutableData<uint8_t>());
uint8_t* dayOfQuarter_data(dayOfQuarter_tensor->MutableData<uint8_t>());
uint16_t* dayOfYear_data(dayOfYear_tensor->MutableData<uint16_t>());
uint16_t* weekOfMonth_data(weekOfMonth_tensor->MutableData<uint16_t>());
uint8_t* quarterOfYear_data(quarterOfYear_tensor->MutableData<uint8_t>());
uint8_t* halfOfYear_data(halfOfYear_tensor->MutableData<uint8_t>());
uint8_t* weekIso_data(weekIso_tensor->MutableData<uint8_t>());
int32_t* yearIso_data(yearIso_tensor->MutableData<int32_t>());
std::string* monthLabel_data(monthLabel_tensor->MutableData<std::string>());
std::string* amPmLabel_data(amPmLabel_tensor->MutableData<std::string>());
std::string* dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData<std::string>());
std::string* holidayName_data(holidayName_tensor->MutableData<std::string>());
uint8_t* isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData<uint8_t>());
// Execute
int64_t const length = input_tensor->Shape().Size();
for (int64_t i = 0; i < length; ++i) {
auto result(transformer.execute(input_data[i]));
year_data[i] = std::move(result.year);
month_data[i] = std::move(result.month);
day_data[i] = std::move(result.day);
hour_data[i] = std::move(result.hour);
minute_data[i] = std::move(result.minute);
second_data[i] = std::move(result.second);
amPm_data[i] = std::move(result.amPm);
hour12_data[i] = std::move(result.hour12);
dayOfWeek_data[i] = std::move(result.dayOfWeek);
dayOfQuarter_data[i] = std::move(result.dayOfQuarter);
dayOfYear_data[i] = std::move(result.dayOfYear);
weekOfMonth_data[i] = std::move(result.weekOfMonth);
quarterOfYear_data[i] = std::move(result.quarterOfYear);
halfOfYear_data[i] = std::move(result.halfOfYear);
weekIso_data[i] = std::move(result.weekIso);
yearIso_data[i] = std::move(result.yearIso);
monthLabel_data[i] = std::move(result.monthLabel);
amPmLabel_data[i] = std::move(result.amPmLabel);
dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel);
holidayName_data[i] = std::move(result.holidayName);
isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff);
}
return Status::OK();
}
};
ONNX_OPERATOR_KERNEL_EX(
DateTimeTransformer,
kMSAutoMLDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT0", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("InputT1", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("OutputT0", DataTypeImpl::GetTensorType<int32_t>())
.TypeConstraint("OutputT1", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("OutputT2", DataTypeImpl::GetTensorType<uint16_t>())
.TypeConstraint("OutputT3", DataTypeImpl::GetTensorType<std::string>()),
DateTimeTransformer);
} // namespace automl
} // namespace onnxruntime

26
onnxruntime/automl_ops/cpu_automl_kernels.cc
View file

@ -1,26 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "automl_ops/cpu_automl_kernels.h"
#include "core/graph/constants.h"
#include "core/framework/data_types.h"
namespace onnxruntime {
namespace automl {
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSAutoMLDomain, 1, DateTimeTransformer);
Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
// add more kernels here
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSAutoMLDomain, 1, DateTimeTransformer)>
};
for (auto& function_table_entry : function_table) {
ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry()));
}
return Status::OK();
}
} // namespace automl
} // namespace onnxruntime

141
onnxruntime/core/graph/automl_ops/automl_defs.cc
View file

@ -1,141 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/graph/constants.h"
#include "core/graph/automl_ops/automl_defs.h"
#include "core/graph/op.h"
#include "onnx/defs/schema.h"
#include "onnx/defs/shape_inference.h"
namespace onnxruntime {
namespace automl {
using ONNX_NAMESPACE::AttributeProto;
using ONNX_NAMESPACE::OpSchema;
using ONNX_NAMESPACE::OPTIONAL;
static const char* DateTimeTransformer_ver1_doc = R"DOC(
Extracts various datetime-related values from a UTC time_point.
C++-style pseudo signature:
TimePoint execute(std::chron::system_clock::time_point const &value);
Examples:
Given a time_point 'value' representing "November 17, 1976 12:27:04PM":
"November 17, 1976 12:27:04PM" => {
"year": 1976,
"month": 11,
"day": 17,
"hour": 12,
"minute": 27,
"second": 04,
"amPm": 2, // PM
"hour12": 12,
"dayOfWeek": 3, // Wednesday
"dayOfQuarter": 48,
"dayOfYear": 321,
"weekOfMonth": 2,
"quarterOfYear": 4,
"halfOfYear": 2,
"weekIso": 47,
"yearIso": 1976,
"monthLabel": "November",
"amPmLabel": "pm",
"dayOfWeekLabel": "Wednesday",
"holidayName": "",
"isPaidTimeOff": 0
}
)DOC";
void RegisterAutoMLSchemas() {
MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer)
.SinceVersion(1)
.SetDomain(kMSAutoMLDomain)
.SetDoc(DateTimeTransformer_ver1_doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"InputT0")
.Input(
1,
"Input",
"The input represents a number of seconds passed since the epoch, suitable to properly construct"
"an instance of std::chrono::system_clock::time_point",
"InputT1")
.Output(0, "year", "No information available", "OutputT0")
.Output(1, "month", "No information available", "OutputT1")
.Output(2, "day", "No information available", "OutputT1")
.Output(3, "hour", "No information available", "OutputT1")
.Output(4, "minute", "No information available", "OutputT1")
.Output(5, "second", "No information available", "OutputT1")
.Output(6, "amPm", "No information available", "OutputT1")
.Output(7, "hour12", "No information available", "OutputT1")
.Output(8, "dayOfWeek", "No information available", "OutputT1")
.Output(9, "dayOfQuarter", "No information available", "OutputT1")
.Output(10, "dayOfYear", "No information available", "OutputT2")
.Output(11, "weekOfMonth", "No information available", "OutputT2")
.Output(12, "quarterOfYear", "No information available", "OutputT1")
.Output(13, "halfOfYear", "No information available", "OutputT1")
.Output(14, "weekIso", "No information available", "OutputT1")
.Output(15, "yearIso", "No information available", "OutputT0")
.Output(16, "monthLabel", "No information available", "OutputT3")
.Output(17, "amPmLabel", "No information available", "OutputT3")
.Output(18, "dayOfWeekLabel", "No information available", "OutputT3")
.Output(19, "holidayName", "No information available", "OutputT3")
.Output(20, "isPaidTimeOff", "No information available", "OutputT1")
.TypeConstraint(
"InputT0",
{"tensor(uint8)"},
"No information is available")
.TypeConstraint(
"InputT1",
{"tensor(int64)"},
"No information is available")
.TypeConstraint(
"OutputT0",
{"tensor(int32)"},
"No information is available")
.TypeConstraint(
"OutputT1",
{"tensor(uint8)"},
"No information is available")
.TypeConstraint(
"OutputT2",
{"tensor(uint16)"},
"No information is available")
.TypeConstraint(
"OutputT3",
{"tensor(string)"},
"No information is available")
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32);
ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16);
ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16);
ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32);
ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
for (size_t i = 0; i < ctx.getNumOutputs(); ++i) {
*ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape();
}
});
}
} // namespace automl
} // namespace onnxruntime

30
onnxruntime/core/graph/automl_ops/automl_defs.h
View file

@ -1,30 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/graph/onnx_protobuf.h"
namespace onnxruntime {
namespace automl {
#define MS_AUTOML_OPERATOR_SCHEMA(name) \
MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) \
MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \
static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \
op_schema_register_once##name##Counter) ONNX_UNUSED = \
ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)
#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) \
MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) \
MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \
static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \
op_schema_register_once##name##Counter) ONNX_UNUSED = \
schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__))
void RegisterAutoMLSchemas();
} // namespace automl
} // namespace onnxruntime

356
onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc Normal file
View file

@ -0,0 +1,356 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/graph/constants.h"
#include "core/graph/featurizers_ops/featurizers_defs.h"
#include "core/graph/op.h"
#include "onnx/defs/schema.h"
#include "onnx/defs/shape_inference.h"
#define MS_AUTOML_OPERATOR_SCHEMA(name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(__COUNTER__, name)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER(Counter, name) MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ(Counter, name) \
static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \
op_schema_register_once##name##Counter) ONNX_UNUSED = \
ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__)
#define MS_AUTOML_OPERATOR_SCHEMA_ELSEWHERE(name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(__COUNTER__, name, schema_func)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_HELPER_ELSEWHERE(Counter, name, schema_func) MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func)
#define MS_AUTOML_OPERATOR_SCHEMA_UNIQ_ELSEWHERE(Counter, name, schema_func) \
static ONNX_NAMESPACE::OpSchemaRegistry::OpSchemaRegisterOnce( \
op_schema_register_once##name##Counter) ONNX_UNUSED = \
schema_func(ONNX_NAMESPACE::OpSchema(#name, __FILE__, __LINE__))
namespace onnxruntime {
namespace featurizers {
using ONNX_NAMESPACE::AttributeProto;
using ONNX_NAMESPACE::OpSchema;
using ONNX_NAMESPACE::OPTIONAL;
// Forward declarations
static void RegisterCatImputerFeaturizerVer1();
static void RegisterDateTimeFeaturizerVer1();
static void RegisterMaxAbsScalarFeaturizerVer1();
static void RegisterStringFeaturizerVer1();
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
void RegisterAutoMLSchemas() {
RegisterCatImputerFeaturizerVer1();
RegisterDateTimeFeaturizerVer1();
RegisterMaxAbsScalarFeaturizerVer1();
RegisterStringFeaturizerVer1();
}
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
void RegisterCatImputerFeaturizerVer1() {
static const char * doc = R"DOC(
Imputes (populates) values with the mode (most common value) encountered during
training. This featurizer supports float and double for most (if not all) frameworks
due to the existance of NaN in those types. Other types require 'optional' support
within the host frameworks and programming languages.
C++-style pseudo signature:
std::float_t execute(std::float_t const &value);
std::double_t execute(std::double_t const &value);
template <typename T> T execute(std::optional<T> const &value);
Examples (where 55.5 is the mode value):
execute(1.0) -> 1.0
execute(NaN) -> 55.5
execute(2.0) -> 2.0
)DOC";
MS_AUTOML_OPERATOR_SCHEMA(CatImputerTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"tensor(uint8)"
)
.Input(
1,
"Input",
"No information is available",
"T"
)
.Output(
0,
"Output",
"No information is available",
"T"
)
.TypeConstraint(
"T",
{"tensor(float)", "tensor(double)", "tensor(string)"},
"No information is available"
)
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
propagateElemTypeFromInputToOutput(ctx, 1, 0);
if (!hasNInputShapes(ctx, 1)) {
return;
}
propagateShapeFromInputToOutput(ctx, 1, 0);
}
)
;
}
void RegisterDateTimeFeaturizerVer1() {
static const char * doc = R"DOC(
Extracts various datetime-related values from a UTC time_point.
C++-style pseudo signature:
TimePoint execute(std::chron::system_clock::time_point const &value);
Examples:
Given a time_point 'value' representing "November 17, 1976 12:27:04PM":
"November 17, 1976 12:27:04PM" => {
"year": 1976,
"month": 11,
"day": 17,
"hour": 12,
"minute": 27,
"second": 04,
"amPm": 2, // PM
"hour12": 12,
"dayOfWeek": 3, // Wednesday
"dayOfQuarter": 48,
"dayOfYear": 321,
"weekOfMonth": 2,
"quarterOfYear": 4,
"halfOfYear": 2,
"weekIso": 47,
"yearIso": 1976,
"monthLabel": "November",
"amPmLabel": "pm",
"dayOfWeekLabel": "Wednesday",
"holidayName": "",
"isPaidTimeOff": 0
}
)DOC";
MS_AUTOML_OPERATOR_SCHEMA(DateTimeTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"tensor(uint8)"
)
.Input(
1,
"Input",
"No information is available",
"tensor(int64)"
)
.Output(0, "year", "No information available", "OutputT0")
.Output(1, "month", "No information available", "OutputT1")
.Output(2, "day", "No information available", "OutputT1")
.Output(3, "hour", "No information available", "OutputT1")
.Output(4, "minute", "No information available", "OutputT1")
.Output(5, "second", "No information available", "OutputT1")
.Output(6, "amPm", "No information available", "OutputT1")
.Output(7, "hour12", "No information available", "OutputT1")
.Output(8, "dayOfWeek", "No information available", "OutputT1")
.Output(9, "dayOfQuarter", "No information available", "OutputT1")
.Output(10, "dayOfYear", "No information available", "OutputT2")
.Output(11, "weekOfMonth", "No information available", "OutputT2")
.Output(12, "quarterOfYear", "No information available", "OutputT1")
.Output(13, "halfOfYear", "No information available", "OutputT1")
.Output(14, "weekIso", "No information available", "OutputT1")
.Output(15, "yearIso", "No information available", "OutputT0")
.Output(16, "monthLabel", "No information available", "OutputT3")
.Output(17, "amPmLabel", "No information available", "OutputT3")
.Output(18, "dayOfWeekLabel", "No information available", "OutputT3")
.Output(19, "holidayName", "No information available", "OutputT3")
.Output(20, "isPaidTimeOff", "No information available", "OutputT1")
.TypeConstraint(
"OutputT0",
{"tensor(int32)"},
"No information is available"
)
.TypeConstraint(
"OutputT1",
{"tensor(uint8)"},
"No information is available"
)
.TypeConstraint(
"OutputT2",
{"tensor(uint16)"},
"No information is available"
)
.TypeConstraint(
"OutputT3",
{"tensor(string)"},
"No information is available"
)
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext &ctx) {
ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32);
ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16);
ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16);
ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32);
ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING);
ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
for(size_t i = 0; i < ctx.getNumOutputs(); ++i) {
*ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape();
}
}
)
;
}
void RegisterMaxAbsScalarFeaturizerVer1() {
static const char * doc = R"DOC(
Scales input based on the maximum absolute value of all data encountered during training.
C++-style pseudo signature:
std::float_t execute(std::uint16_t value);
std::double_t execute(std::uint32_t value);
Examples:
Given a training set of [1.0, -2.0, 3.0, -4.0], where 4.0 is the absolute value of the
maximum value encountered...
execute(1.0) -> 1.0 / 4.0
execute(-4.0) -> -4.0 / 4.0
execute(100.0) -> 100 / 4.0
)DOC";
MS_AUTOML_OPERATOR_SCHEMA(MaxAbsScalarTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"tensor(uint8)"
)
.Input(
1,
"Input",
"No information is available",
"InputT"
)
.Output(
0,
"Output",
"No information is available",
"OutputT"
)
.TypeConstraint(
"InputT",
{"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"},
"No information is available"
)
.TypeConstraint(
"OutputT",
{"tensor(float)", "tensor(double)"},
"No information is available"
)
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
auto input_elem_type = ctx.getInputType(1)->tensor_type().elem_type();
if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT);
} else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 ||
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) {
ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE);
}
*ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape();
}
)
;
}
void RegisterStringFeaturizerVer1() {
static const char * doc = R"DOC(
Converts the input into a string representation based on the input's type.
C++-style pseudo signature:
template <typename T> std::string execute(T const &value);
Examples:
execute(1) -> "1"
execute(3.14) -> "3.14"
)DOC";
MS_AUTOML_OPERATOR_SCHEMA(StringTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"tensor(uint8)"
)
.Input(
1,
"Input",
"No information is available",
"InputT"
)
.Output(
0,
"Output",
"No information is available",
"tensor(string)"
)
.TypeConstraint(
"InputT",
{"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"},
"No information is available"
)
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 0);
*ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape();
}
)
;
}
} // namespace featurizers
} // namespace onnxruntime

12
onnxruntime/core/graph/featurizers_ops/featurizers_defs.h Normal file
View file

@ -0,0 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
namespace onnxruntime {
namespace featurizers {
void RegisterAutoMLSchemas(void);
} // namespace featurizers
} // namespace onnxruntime

8
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -9,8 +9,8 @@
#include "contrib_ops/cpu_contrib_kernels.h"
#endif
#ifdef MICROSOFT_AUTOML
#include "automl_ops/cpu_automl_kernels.h"
#ifdef ML_FEATURIZERS
#include "featurizers_ops/cpu_featurizers_kernels.h"
#endif
#include "core/framework/compute_capability.h"
@ -1270,8 +1270,8 @@ static Status RegisterCPUKernels(KernelRegistry& kernel_registry) {
#ifndef DISABLE_CONTRIB_OPS
ORT_RETURN_IF_ERROR(::onnxruntime::contrib::RegisterCpuContribKernels(kernel_registry));
#endif
#ifdef MICROSOFT_AUTOML
ORT_RETURN_IF_ERROR(::onnxruntime::automl::RegisterCpuAutoMLKernels(kernel_registry));
#ifdef ML_FEATURIZERS
ORT_RETURN_IF_ERROR(::onnxruntime::featurizers::RegisterCpuAutoMLKernels(kernel_registry));
#endif
return Status::OK();
}

10
onnxruntime/core/session/environment.cc
View file

@ -10,8 +10,8 @@
#ifndef DISABLE_CONTRIB_OPS
#include "core/graph/contrib_ops/contrib_defs.h"
#endif
#ifdef MICROSOFT_AUTOML
#include "core/graph/automl_ops/automl_defs.h"
#ifdef ML_FEATURIZERS
#include "core/graph/featurizers_ops/featurizers_defs.h"
#endif
#ifdef USE_DML
#include "core/graph/dml_ops/dml_defs.h"
@ -45,7 +45,7 @@ Status Environment::Initialize() {
std::call_once(schemaRegistrationOnceFlag, []() {
ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDomain, 1, 1);
ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSNchwcDomain, 1, 1);
ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSAutoMLDomain, 1, 1);
ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSFeaturizersDomain, 1, 1);
#ifdef USE_DML
ONNX_NAMESPACE::OpSchemaRegistry::DomainToVersionRange::Instance().AddDomainToVersion(onnxruntime::kMSDmlDomain, 1, 1);
#endif
@ -54,8 +54,8 @@ Status Environment::Initialize() {
#ifndef DISABLE_CONTRIB_OPS
contrib::RegisterContribSchemas();
#endif
#ifdef MICROSOFT_AUTOML
automl::RegisterAutoMLSchemas();
#ifdef ML_FEATURIZERS
featurizers::RegisterAutoMLSchemas();
#endif
#ifdef USE_DML
dml::RegisterDmlSchemas();

97
onnxruntime/featurizers_ops/cpu/CatImputerTransformer.cc Normal file
View file

@ -0,0 +1,97 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/CatImputerFeaturizer.h"
namespace feat = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace featurizers {
template <typename T>
struct OutputTypeMapper {};
template <>
struct OutputTypeMapper<float_t> { using type = float_t; };
template <>
struct OutputTypeMapper<double_t> { using type = double_t; };
template <>
struct OutputTypeMapper<std::string> { using type = std::string; };
inline float_t const& PreprocessOptional(float_t const& value) { return value; }
inline double_t const& PreprocessOptional(double_t const& value) { return value; }
inline nonstd::optional<std::string> PreprocessOptional(std::string value) {
return value.empty() ? nonstd::optional<std::string>() : nonstd::optional<std::string>(std::move(value));
}
template <typename T>
class CatImputerTransformer final : public OpKernel {
public:
explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* ctx) const override {
// Create the transformer
feat::CatImputerTransformer<T> transformer(
[ctx](void) {
const auto* state_tensor(ctx->Input<Tensor>(0));
const uint8_t* const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return feat::CatImputerTransformer<T>(archive);
}());
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const T* input_data(input_tensor->Data<T>());
// Prepare the output
Tensor* output_tensor(ctx->Output(0, input_tensor->Shape()));
typename OutputTypeMapper<T>::type* output_data(output_tensor->MutableData<typename OutputTypeMapper<T>::type>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for (int64_t i = 0; i < length; ++i) {
output_data[i] = transformer.execute(PreprocessOptional(input_data[i]));
}
return Status::OK();
}
};
ONNX_OPERATOR_TYPED_KERNEL_EX(
CatImputerTransformer,
kMSFeaturizersDomain,
1,
float_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<float_t>()),
CatImputerTransformer<float_t>);
ONNX_OPERATOR_TYPED_KERNEL_EX(
CatImputerTransformer,
kMSFeaturizersDomain,
1,
double_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<double_t>()),
CatImputerTransformer<double_t>);
ONNX_OPERATOR_TYPED_KERNEL_EX(
CatImputerTransformer,
kMSFeaturizersDomain,
1,
string,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<std::string>()),
CatImputerTransformer<std::string>);
} // namespace featurizers
} // namespace onnxruntime

128
onnxruntime/featurizers_ops/cpu/DateTimeTransformer.cc Normal file
View file

@ -0,0 +1,128 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/DateTimeFeaturizer.h"
namespace feat = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace featurizers {
class DateTimeTransformer final : public OpKernel {
public:
explicit DateTimeTransformer(const OpKernelInfo &info) : OpKernel(info) {
}
Status Compute(OpKernelContext *ctx) const override {
// Create the transformer
feat::DateTimeTransformer transformer(
[ctx](void) {
const auto * state_tensor(ctx->Input<Tensor>(0));
const uint8_t * const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return feat::DateTimeTransformer(archive);
}()
);
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const std::int64_t * input_data(input_tensor->Data<std::int64_t>());
// Prepare the output
Tensor * year_tensor(ctx->Output(0, input_tensor->Shape()));
Tensor * month_tensor(ctx->Output(1, input_tensor->Shape()));
Tensor * day_tensor(ctx->Output(2, input_tensor->Shape()));
Tensor * hour_tensor(ctx->Output(3, input_tensor->Shape()));
Tensor * minute_tensor(ctx->Output(4, input_tensor->Shape()));
Tensor * second_tensor(ctx->Output(5, input_tensor->Shape()));
Tensor * amPm_tensor(ctx->Output(6, input_tensor->Shape()));
Tensor * hour12_tensor(ctx->Output(7, input_tensor->Shape()));
Tensor * dayOfWeek_tensor(ctx->Output(8, input_tensor->Shape()));
Tensor * dayOfQuarter_tensor(ctx->Output(9, input_tensor->Shape()));
Tensor * dayOfYear_tensor(ctx->Output(10, input_tensor->Shape()));
Tensor * weekOfMonth_tensor(ctx->Output(11, input_tensor->Shape()));
Tensor * quarterOfYear_tensor(ctx->Output(12, input_tensor->Shape()));
Tensor * halfOfYear_tensor(ctx->Output(13, input_tensor->Shape()));
Tensor * weekIso_tensor(ctx->Output(14, input_tensor->Shape()));
Tensor * yearIso_tensor(ctx->Output(15, input_tensor->Shape()));
Tensor * monthLabel_tensor(ctx->Output(16, input_tensor->Shape()));
Tensor * amPmLabel_tensor(ctx->Output(17, input_tensor->Shape()));
Tensor * dayOfWeekLabel_tensor(ctx->Output(18, input_tensor->Shape()));
Tensor * holidayName_tensor(ctx->Output(19, input_tensor->Shape()));
Tensor * isPaidTimeOff_tensor(ctx->Output(20, input_tensor->Shape()));
int32_t * year_data(year_tensor->MutableData<int32_t>());
uint8_t * month_data(month_tensor->MutableData<uint8_t>());
uint8_t * day_data(day_tensor->MutableData<uint8_t>());
uint8_t * hour_data(hour_tensor->MutableData<uint8_t>());
uint8_t * minute_data(minute_tensor->MutableData<uint8_t>());
uint8_t * second_data(second_tensor->MutableData<uint8_t>());
uint8_t * amPm_data(amPm_tensor->MutableData<uint8_t>());
uint8_t * hour12_data(hour12_tensor->MutableData<uint8_t>());
uint8_t * dayOfWeek_data(dayOfWeek_tensor->MutableData<uint8_t>());
uint8_t * dayOfQuarter_data(dayOfQuarter_tensor->MutableData<uint8_t>());
uint16_t * dayOfYear_data(dayOfYear_tensor->MutableData<uint16_t>());
uint16_t * weekOfMonth_data(weekOfMonth_tensor->MutableData<uint16_t>());
uint8_t * quarterOfYear_data(quarterOfYear_tensor->MutableData<uint8_t>());
uint8_t * halfOfYear_data(halfOfYear_tensor->MutableData<uint8_t>());
uint8_t * weekIso_data(weekIso_tensor->MutableData<uint8_t>());
int32_t * yearIso_data(yearIso_tensor->MutableData<int32_t>());
std::string * monthLabel_data(monthLabel_tensor->MutableData<std::string>());
std::string * amPmLabel_data(amPmLabel_tensor->MutableData<std::string>());
std::string * dayOfWeekLabel_data(dayOfWeekLabel_tensor->MutableData<std::string>());
std::string * holidayName_data(holidayName_tensor->MutableData<std::string>());
uint8_t * isPaidTimeOff_data(isPaidTimeOff_tensor->MutableData<uint8_t>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for(int64_t i = 0; i < length; ++i) {
auto result(transformer.execute(input_data[i]));
year_data[i] = std::move(result.year);
month_data[i] = std::move(result.month);
day_data[i] = std::move(result.day);
hour_data[i] = std::move(result.hour);
minute_data[i] = std::move(result.minute);
second_data[i] = std::move(result.second);
amPm_data[i] = std::move(result.amPm);
hour12_data[i] = std::move(result.hour12);
dayOfWeek_data[i] = std::move(result.dayOfWeek);
dayOfQuarter_data[i] = std::move(result.dayOfQuarter);
dayOfYear_data[i] = std::move(result.dayOfYear);
weekOfMonth_data[i] = std::move(result.weekOfMonth);
quarterOfYear_data[i] = std::move(result.quarterOfYear);
halfOfYear_data[i] = std::move(result.halfOfYear);
weekIso_data[i] = std::move(result.weekIso);
yearIso_data[i] = std::move(result.yearIso);
monthLabel_data[i] = std::move(result.monthLabel);
amPmLabel_data[i] = std::move(result.amPmLabel);
dayOfWeekLabel_data[i] = std::move(result.dayOfWeekLabel);
holidayName_data[i] = std::move(result.holidayName);
isPaidTimeOff_data[i] = std::move(result.isPaidTimeOff);
}
return Status::OK();
}
};
ONNX_OPERATOR_KERNEL_EX(
DateTimeTransformer,
kMSFeaturizersDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("OutputT0", DataTypeImpl::GetTensorType<int32_t>())
.TypeConstraint("OutputT1", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("OutputT2", DataTypeImpl::GetTensorType<uint16_t>())
.TypeConstraint("OutputT3", DataTypeImpl::GetTensorType<std::string>()),
DateTimeTransformer
);
} // namespace featurizers
} // namespace onnxruntime

175
onnxruntime/featurizers_ops/cpu/MaxAbsScalarTransformer.cc Normal file
View file

@ -0,0 +1,175 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/MaxAbsScalarFeaturizer.h"
namespace feat = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace featurizers {
template <typename T> struct OutputTypeMapper {};
template <> struct OutputTypeMapper<int8_t> { using type = float_t; };
template <> struct OutputTypeMapper<int16_t> { using type = float_t; };
template <> struct OutputTypeMapper<uint8_t> { using type = float_t; };
template <> struct OutputTypeMapper<uint16_t> { using type = float_t; };
template <> struct OutputTypeMapper<float_t> { using type = float_t; };
template <> struct OutputTypeMapper<int32_t> { using type = double_t; };
template <> struct OutputTypeMapper<int64_t> { using type = double_t; };
template <> struct OutputTypeMapper<uint32_t> { using type = double_t; };
template <> struct OutputTypeMapper<uint64_t> { using type = double_t; };
template <> struct OutputTypeMapper<double_t> { using type = double_t; };
template <typename InputT>
class MaxAbsScalarTransformer final : public OpKernel {
public:
explicit MaxAbsScalarTransformer(const OpKernelInfo &info) : OpKernel(info) {
}
Status Compute(OpKernelContext *ctx) const override {
// Create the transformer
feat::MaxAbsScalarTransformer<InputT, typename OutputTypeMapper<InputT>::type> transformer(
[ctx](void) {
const auto * state_tensor(ctx->Input<Tensor>(0));
const uint8_t * const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return feat::MaxAbsScalarTransformer<InputT, typename OutputTypeMapper<InputT>::type>(archive);
}()
);
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const InputT * input_data(input_tensor->Data<InputT>());
// Prepare the output
Tensor * output_tensor(ctx->Output(0, input_tensor->Shape()));
typename OutputTypeMapper<InputT>::type * output_data(output_tensor->MutableData<typename OutputTypeMapper<InputT>::type>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for(int64_t i = 0; i < length; ++i) {
output_data[i] = transformer.execute(input_data[i]);
}
return Status::OK();
}
};
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
int8_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int8_t>()),
MaxAbsScalarTransformer<int8_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
int16_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int16_t>()),
MaxAbsScalarTransformer<int16_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
uint8_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint8_t>()),
MaxAbsScalarTransformer<uint8_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
uint16_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint16_t>()),
MaxAbsScalarTransformer<uint16_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
float_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<float_t>()),
MaxAbsScalarTransformer<float_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
int32_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int32_t>()),
MaxAbsScalarTransformer<int32_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
int64_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int64_t>()),
MaxAbsScalarTransformer<int64_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
uint32_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint32_t>()),
MaxAbsScalarTransformer<uint32_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
uint64_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint64_t>()),
MaxAbsScalarTransformer<uint64_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
MaxAbsScalarTransformer,
kMSFeaturizersDomain,
1,
double_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<double_t>()),
MaxAbsScalarTransformer<double_t>
);
} // namespace featurizers
} // namespace onnxruntime

185
onnxruntime/featurizers_ops/cpu/StringTransformer.cc Normal file
View file

@ -0,0 +1,185 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/StringFeaturizer.h"
namespace feat = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace featurizers {
template <typename InputT>
class StringTransformer final : public OpKernel {
public:
explicit StringTransformer(const OpKernelInfo &info) : OpKernel(info) {
}
Status Compute(OpKernelContext *ctx) const override {
// Create the transformer
feat::StringTransformer<InputT> transformer(
[ctx](void) {
const auto * state_tensor(ctx->Input<Tensor>(0));
const uint8_t * const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return feat::StringTransformer<InputT>(archive);
}()
);
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const InputT * input_data(input_tensor->Data<InputT>());
// Prepare the output
Tensor * output_tensor(ctx->Output(0, input_tensor->Shape()));
std::string * output_data(output_tensor->MutableData<std::string>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for(int64_t i = 0; i < length; ++i) {
output_data[i] = transformer.execute(input_data[i]);
}
return Status::OK();
}
};
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
int8_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int8_t>()),
StringTransformer<int8_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
int16_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int16_t>()),
StringTransformer<int16_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
int32_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int32_t>()),
StringTransformer<int32_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
int64_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<int64_t>()),
StringTransformer<int64_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
uint8_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint8_t>()),
StringTransformer<uint8_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
uint16_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint16_t>()),
StringTransformer<uint16_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
uint32_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint32_t>()),
StringTransformer<uint32_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
uint64_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<uint64_t>()),
StringTransformer<uint64_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
float_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<float_t>()),
StringTransformer<float_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
double_t,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<double_t>()),
StringTransformer<double_t>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
bool,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<bool>()),
StringTransformer<bool>
);
ONNX_OPERATOR_TYPED_KERNEL_EX(
StringTransformer,
kMSFeaturizersDomain,
1,
string,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("InputT", DataTypeImpl::GetTensorType<std::string>()),
StringTransformer<std::string>
);
} // namespace featurizers
} // namespace onnxruntime

78
onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc Normal file
View file

@ -0,0 +1,78 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "featurizers_ops/cpu_featurizers_kernels.h"
#include "core/graph/constants.h"
#include "core/framework/data_types.h"
namespace onnxruntime {
namespace featurizers {
// Forward declarations
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, CatImputerTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, CatImputerTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, MaxAbsScalarTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer);
Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, CatImputerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, CatImputerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double_t, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer)>
};
for (auto& function_table_entry : function_table) {
ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry()));
}
return Status::OK();
}
} // namespace featurizers
} // namespace onnxruntime

8
onnxruntime/automl_ops/cpu_automl_kernels.h → onnxruntime/featurizers_ops/cpu_featurizers_kernels.h
View file

@ -7,7 +7,9 @@
#include "core/framework/kernel_registry.h"
namespace onnxruntime {
namespace automl {
namespace featurizers {
Status RegisterCpuAutoMLKernels(KernelRegistry& kernel_registry);
} // namespace automl
} // namespace onnxruntime
} // namespace featurizers
} // namespace onnxruntime

64
onnxruntime/test/featurizers_ops/categoryimputer_test.cc Normal file
View file

@ -0,0 +1,64 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Featurizers/CatImputerFeaturizer.h"
namespace dft = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace test {
TEST(CategoryImputer, Float_values) {
OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State from when the transformer was trained. Corresponds to a
// most frequent value during training of 1.5
test.AddInput<uint8_t>("State", {4}, {0, 0, 192, 63});
// We are adding a scalar Tensor in this instance
test.AddInput<float_t>("Input", {5}, {1, std::nanf("1"), std::nanf("1"), 2, std::nanf("1")});
// Expected output.
test.AddOutput<float_t>("Output", {5}, {1, 1.5, 1.5, 2, 1.5});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(CategoryImputer, Double_values) {
OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State from when the transformer was trained. Corresponds to a
// most frequent value during training of 1.5
test.AddInput<uint8_t>("State", {8}, {0, 0, 0, 0, 0, 0, 248, 63});
// We are adding a scalar Tensor in this instance
test.AddInput<double_t>("Input", {5}, {1, std::nan("1"), std::nan("1"), 2, std::nan("1")});
// Expected output.
test.AddOutput<double_t>("Output", {5}, {1, 1.5, 1.5, 2, 1.5});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(CategoryImputer, String_values) {
OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State from when the transformer was trained. Corresponds to a
// most frequent value during training of "one"
test.AddInput<uint8_t>("State", {7}, {3, 0, 0, 0, 111, 110, 101});
// We are adding a scalar Tensor in this instance
test.AddInput<std::string>("Input", {5}, {"ONE", "", "FIVE", "", "NINE"});
// Expected output.
test.AddOutput<std::string>("Output", {5}, {"ONE", "one", "FIVE", "one", "NINE"});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
} // namespace test
} // namespace onnxruntime

8
onnxruntime/test/automl_ops/datetimetransformer_test.cc → onnxruntime/test/featurizers_ops/datetimetransformer_test.cc
View file

@ -15,7 +15,7 @@ namespace test {
TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) {
const time_t date = 217081624;
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {4}, {0, 0, 0, 0});
@ -77,7 +77,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) {
TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) {
const time_t date = 217081625;
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {4}, {0, 0, 0, 0});
@ -139,7 +139,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27
const time_t date1 = 217081625;
const time_t date2 = 217081624;
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {4}, {0, 0, 0, 0});
@ -226,7 +226,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27
TEST(DateTimeTransformer, Future_2025_June_30) {
const time_t date = 1751241600;
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSAutoMLDomain);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {4}, {0, 0, 0, 0});

48
onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc Normal file
View file

@ -0,0 +1,48 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Featurizers/MaxAbsScalarFeaturizer.h"
namespace dft = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace test {
TEST(MaxAbsScaler, Int8_values) {
OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State from when the transformer was trained. Corresponds to Version 1 and a
// scale of 0
test.AddInput<uint8_t>("State", {4}, {0, 0, 128, 64});
// We are adding a scalar Tensor in this instance
test.AddInput<int8_t>("X", {5}, {-4,3,0,2,-1});
// Expected output.
test.AddOutput<float_t>("ScaledValues", {5}, {-1,.75,0,.5,-.25});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(MaxAbsScaler, Double_values) {
OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State from when the transformer was trained. Corresponds to Version 1 and a
// scale of 0
test.AddInput<uint8_t>("State", {8}, {0, 0, 0, 0, 0, 0, 16, 64});
// We are adding a scalar Tensor in this instance
test.AddInput<double_t>("X", {5}, {-4, 3, 0, 2, -1});
// Expected output.
test.AddOutput<double_t>("ScaledValues", {5}, {-1, .75, 0, .5, -.25});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
} // namespace test
} // namespace onnxruntime

75
onnxruntime/test/featurizers_ops/stringtransformer_test.cc Normal file
View file

@ -0,0 +1,75 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Featurizers/StringFeaturizer.h"
namespace dft = Microsoft::Featurizer::Featurizers;
namespace onnxruntime {
namespace test {
TEST(StringTransformer, Integer_values) {
OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State input is needed, but no actual state is required.
test.AddInput<uint8_t>("State", {0}, {});
// We are adding a scalar Tensor in this instance
test.AddInput<int64_t>("Input", {5}, {1, 3, 5, 7, 9});
// Expected output.
test.AddOutput<std::string>("Output", {5}, {"1", "3", "5", "7", "9"});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(StringTransformer, Double_values) {
OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State input is needed, but no actual state is required.
test.AddInput<uint8_t>("State", {0}, {});
// We are adding a scalar Tensor in this instance
test.AddInput<double>("Input", {5}, {1, 3, 5, 7, 9});
// Expected output.
test.AddOutput<std::string>("Output", {5}, {"1.000000", "3.000000", "5.000000", "7.000000", "9.000000"});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(StringTransformer, Bool_values) {
OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State input is needed, but no actual state is required.
test.AddInput<uint8_t>("State", {0}, {});
// We are adding a scalar Tensor in this instance
test.AddInput<bool>("Input", {5}, {true, false, false, false, true});
// Expected output.
test.AddOutput<std::string>("Output", {5}, {"True", "False", "False", "False", "True"});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(StringTransformer, String_values) {
OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// State input is needed, but no actual state is required.
test.AddInput<uint8_t>("State", {0}, {});
// We are adding a scalar Tensor in this instance
test.AddInput<std::string>("Input", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"});
// Expected output.
test.AddOutput<std::string>("Output", {5}, {"ONE", "three", "FIVE", "SeVeN", "NINE"});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
} // namespace test
} // namespace onnxruntime

8
onnxruntime/test/opaque_api/test_opaque_api.cc
View file

@ -107,7 +107,7 @@ class OpaqueCApiTestKernel final : public OpKernel {
ONNX_OPERATOR_KERNEL_EX(
OpaqueCApiTestKernel,
kMSAutoMLDomain,
kMSFeaturizersDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
@ -131,7 +131,7 @@ static void RegisterCustomKernel() {
// Registry the schema
ONNX_TEST_OPERATOR_SCHEMA(OpaqueCApiTestKernel)
.SetDoc("Replace all of h chars to _ in the original string contained within experimental type")
.SetDomain(onnxruntime::kMSAutoMLDomain)
.SetDomain(onnxruntime::kMSFeaturizersDomain)
.SinceVersion(1)
.Input(
0,
@ -153,7 +153,7 @@ static void RegisterCustomKernel() {
// Register kernel directly to KernelRegistry
// because we can not create custom ops with Opaque types
// as input
BuildKernelCreateInfoFn fn = BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSAutoMLDomain, 1, OpaqueCApiTestKernel)>;
BuildKernelCreateInfoFn fn = BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, OpaqueCApiTestKernel)>;
auto kernel_registry = CPUExecutionProvider(CPUExecutionProviderInfo()).GetKernelRegistry();
kernel_registry->Register(fn());
}
@ -179,7 +179,7 @@ std::string CreateModel() {
outputs.push_back(&output_arg);
auto& node = graph.AddNode("OpaqueCApiTestKernel", "OpaqueCApiTestKernel", "Replace all h to underscore",
inputs, outputs, nullptr, onnxruntime::kMSAutoMLDomain);
inputs, outputs, nullptr, onnxruntime::kMSFeaturizersDomain);
node.SetExecutionProviderType(onnxruntime::kCpuExecutionProvider);
}
EXPECT_TRUE(graph.Resolve().IsOK());

4
tools/ci_build/build.py
View file

@ -130,7 +130,7 @@ Use the individual flags to only run the specified stages.
parser.add_argument("--use_dnnl", action='store_true', help="Build with DNNL.")
parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.")
parser.add_argument("--use_gemmlowp", action='store_true', help="Build with gemmlowp for quantized gemm.")
parser.add_argument("--use_automl", action='store_true', help="Build with AutoML support.")
parser.add_argument("--use_featurizers", action='store_true', help="Build with ML Featurizer support.")
parser.add_argument("--use_ngraph", action='store_true', help="Build with nGraph.")
parser.add_argument("--use_openvino", nargs="?", const="CPU_FP32",
choices=["CPU_FP32","GPU_FP32","GPU_FP16","VAD-M_FP16","MYRIAD_FP16","VAD-F_FP32"], help="Build with OpenVINO for specific hardware.")
@ -290,7 +290,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
"-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"),
"-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"),
"-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""),
"-Donnxruntime_USE_AUTOML=" + ("ON" if args.use_automl else "OFF"),
"-Donnxruntime_USE_FEATURIZERS=" + ("ON" if args.use_featurizers else "OFF"),
"-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""),
"-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"),
"-Donnxruntime_USE_MIMALLOC=" + ("ON" if args.use_mimalloc else "OFF"),

6
tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
View file

@ -40,7 +40,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
@ -101,7 +101,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
@ -174,7 +174,7 @@ jobs:
displayName: 'BUILD'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --enable_onnx_tests --parallel'
workingDirectory: '$(Build.BinariesDirectory)'
- task: CopyFiles@2

4
tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
View file

@ -15,7 +15,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
@ -39,7 +39,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:

2
tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
View file

@ -3,7 +3,7 @@ jobs:
parameters:
AgentPool : 'Linux-CPU'
JobName: 'Linux_CI_Dev'
BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_automl --build_wheel --build_java --enable_language_interop_ops"'
BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --use_featurizers --build_wheel --build_java --enable_language_interop_ops"'
DoNugetPack: 'false'
ArtifactName: 'drop-linux'
TimeoutInMinutes: 120

2
tools/ci_build/github/azure-pipelines/mac-ci-pipeline.yml
View file

@ -3,4 +3,4 @@ jobs:
parameters:
AgentPool : 'Hosted macOS High Sierra'
DoNugetPack: 'false'
BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_automl --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo'
BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --build_wheel --skip_submodule_sync --use_featurizers --parallel --build_shared_lib --build_java --enable_language_interop_ops --enable_onnx_tests --config Debug RelWithDebInfo'

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml
View file

@ -39,7 +39,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml
View file

@ -76,7 +76,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests --disable_contrib_ops && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
View file

@ -55,7 +55,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
View file

@ -43,7 +43,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
- script: |
set -e -x
mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64

2
tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
View file

@ -42,7 +42,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --use_dnnl --use_openmp --build_shared_lib --enable_onnx_tests --build_java'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1

2
tools/ci_build/github/azure-pipelines/win-x86-ci-pipeline.yml
View file

@ -42,7 +42,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --x86 --use_openmp --build_shared_lib --enable_onnx_tests'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_featurizers --x86 --use_openmp --build_shared_lib --enable_onnx_tests'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1