saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add OneHotEncoder and HashOneHotEncoder kernels. (#2830)

Browse source 
Add defs and imlementation for OneHotEncoders, adjuist date_time_transformer kernel and test.
  Add OneHotEncoder kernel test.
  Add HashOneHotVectorizerTransformer unit test.
  This does not link due to multiple definitions of functions
  that are included into header from a CPP file.
This commit is contained in:
Dmitri Smirnov 2020-01-13 17:58:33 -08:00 committed by GitHub
parent 723cf83793
commit 120433c29d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 728 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cgmanifest.json
View file

@ -450,7 +450,7 @@
{
"component": {
"git": {
"commitHash": "573070aeeb77e267da2579ac1d75d92c688bbe97",
"commitHash": "ebec32ef06859b6399bf8854f18b91158c87760b",
"repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git"
},
"type": "git"

2
cmake/external/featurizers.cmake vendored
View file

@ -3,7 +3,7 @@
# This source code should not depend on the onnxruntime and may be built independently
set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git")
set(featurizers_TAG "573070aeeb77e267da2579ac1d75d92c688bbe97")
set(featurizers_TAG "ebec32ef06859b6399bf8854f18b91158c87760b")
set(featurizers_pref FeaturizersLibrary)
set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref})

367
onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc
View file

@ -34,11 +34,13 @@ using ONNX_NAMESPACE::OPTIONAL;
// Forward declarations
static void RegisterCatImputerFeaturizerVer1();
static void RegisterDateTimeFeaturizerVer1();
static void RegisterHashOneHotVectorizerFeaturizerVer1();
static void RegisterImputationMarkerFeaturizerVer1();
static void RegisterLabelEncoderFeaturizerVer1();
static void RegisterMaxAbsScalarFeaturizerVer1();
static void RegisterMinMaxScalarFeaturizerVer1();
static void RegisterMissingDummiesFeaturizerVer1();
static void RegisterOneHotEncoderFeaturizerVer1();
static void RegisterRobustScalarFeaturizerVer1();
static void RegisterStringFeaturizerVer1();
static void RegisterTimeSeriesImputerFeaturizerVer1();
@ -49,11 +51,13 @@ static void RegisterTimeSeriesImputerFeaturizerVer1();
void RegisterMSFeaturizersSchemas() {
RegisterCatImputerFeaturizerVer1();
RegisterDateTimeFeaturizerVer1();
RegisterHashOneHotVectorizerFeaturizerVer1();
RegisterImputationMarkerFeaturizerVer1();
RegisterLabelEncoderFeaturizerVer1();
RegisterMaxAbsScalarFeaturizerVer1();
RegisterMinMaxScalarFeaturizerVer1();
RegisterMissingDummiesFeaturizerVer1();
RegisterOneHotEncoderFeaturizerVer1();
RegisterRobustScalarFeaturizerVer1();
RegisterStringFeaturizerVer1();
RegisterTimeSeriesImputerFeaturizerVer1();
@ -70,15 +74,15 @@ void RegisterCatImputerFeaturizerVer1() {
within the host frameworks and programming languages.
C++-style pseudo signature:
std::float_t execute(std::float_t const &value);
std::double_t execute(std::double_t const &value);
float execute(float const &value);
double execute(double const &value);
template <typename T> T execute(std::optional<T> const &value);
Examples (where 55.5 is the mode value):
execute(1.0) -> 1.0
execute(NaN) -> 55.5
execute(2.0) -> 2.0
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(CatImputerTransformer)
.SinceVersion(1)
@ -121,7 +125,7 @@ void RegisterDateTimeFeaturizerVer1() {
Extracts various datetime-related values from a UTC time_point.
C++-style pseudo signature:
TimePoint execute(std::chron::system_clock::time_point const &value);
TimePoint execute(std::chrono::system_clock::time_point const &value);
Examples:
Given a time_point 'value' representing "November 17, 1976 12:27:04PM":
@ -149,7 +153,7 @@ void RegisterDateTimeFeaturizerVer1() {
"holidayName": "",
"isPaidTimeOff": 0
}
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(DateTimeTransformer)
.SinceVersion(1)
@ -209,52 +213,191 @@ void RegisterDateTimeFeaturizerVer1() {
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
const bool has_shape = hasInputShape(ctx, 1);
for (int output = 0; output < 21; ++output) {
switch (output) {
case 0:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, output);
break;
case 1: // fall through
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
case 9:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output);
break;
case 10: // fall through
case 11:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, output);
break;
case 12: // fall through
case 13:
case 14:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output);
break;
case 15:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, output);
break;
case 16:
case 17:
case 18:
case 19:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, output);
break;
case 20:
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output);
break;
default:
assert(false);
break;
}
if (has_shape) {
propagateShapeFromInputToOutput(ctx, 1, output);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, 0);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 0);
}
});
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 1);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 2);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 2);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 3);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 3);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 4);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 4);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 5);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 5);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 6);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 6);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 7);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 7);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 8);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 8);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 9);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 9);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, 10);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 10);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, 11);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 11);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 12);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 12);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 13);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 13);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 14);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 14);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, 15);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 15);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 16);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 16);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 17);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 17);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 18);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 18);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 19);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 19);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 20);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 20);
}
}
);
}
void RegisterHashOneHotVectorizerFeaturizerVer1() {
static const char* doc = R"DOC(
Hashes the input to a categorical value, then produces a one hot encoded vector
based on that value.
C++-style pseudo signature:
template <typename T> HashOneHotVectorizerStruct execute(T const &value);
Examples:
Assuming the hashing algorithm...
"A" -> 1
"B" -> 2
"C" -> 5
and 'numCols' set to 8:
execute("A") -> [1, 0, 0, 0, 0, 0, 0, 0]
execute("B") -> [0, 1, 0, 0, 0, 0, 0, 0]
execute("C") -> [0, 0, 0, 0, 1, 0, 0, 0]
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(HashOneHotVectorizerTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"T0")
.Input(
1,
"Input",
"No information is available",
"InputT")
.Output(0, "NumElements", "No information available", "OutputT0")
.Output(1, "Value", "No information available", "OutputT1")
.Output(2, "Index", "No information available", "OutputT0")
.TypeConstraint(
"T0",
{"tensor(uint8)"},
"No information is available")
.TypeConstraint(
"InputT",
{"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"},
"No information is available")
.TypeConstraint(
"OutputT0",
{"tensor(uint64)"},
"No information is available")
.TypeConstraint(
"OutputT1",
{"tensor(uint8)"},
"No information is available")
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
const bool has_shape = hasInputShape(ctx, 1);
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 0);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 0);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 1);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 2);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 2);
}
}
);
}
void RegisterImputationMarkerFeaturizerVer1() {
@ -262,17 +405,17 @@ void RegisterImputationMarkerFeaturizerVer1() {
Returns true if the input is null, false if it is not.
C++-style pseudo signature:
bool execute(std::float_t const &value);
bool execute(std::double_t const &value);
bool execute(float const &value);
bool execute(double const &value);
template <typename T> bool execute(std::optional<T> const &value);
Examples:
3.0 -> false
NaN -> true
"foo" -> false
std::optional<std::string>() -> true
std::optional<std::string>("bar") -> false
)DOC";
std::optional<string>() -> true
std::optional<string>("bar") -> false
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(ImputationMarkerTransformer)
.SinceVersion(1)
@ -315,7 +458,7 @@ void RegisterLabelEncoderFeaturizerVer1() {
Returns a unique id for the input based on all values encountered during training.
C++-style pseudo signature:
template <typename T> std::uint32_t execute(T const &value);
template <typename T> uint32 execute(T const &value);
Examples:
Assuming the training data of ["A", "B", "C"]...
@ -324,7 +467,7 @@ void RegisterLabelEncoderFeaturizerVer1() {
execute("B") -> 2
execute("C") -> 3
execute("This value was not seen during training") -> 0
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(LabelEncoderTransformer)
.SinceVersion(1)
@ -367,8 +510,8 @@ void RegisterMaxAbsScalarFeaturizerVer1() {
Scales input based on the maximum absolute value of all data encountered during training.
C++-style pseudo signature:
std::float_t execute(std::uint16_t value);
std::double_t execute(std::uint32_t value);
float execute(uint16 value);
double execute(uint32 value);
Examples:
Given a training set of [1.0, -2.0, 3.0, -4.0], where 4.0 is the absolute value of the
@ -377,7 +520,7 @@ void RegisterMaxAbsScalarFeaturizerVer1() {
execute(1.0) -> 1.0 / 4.0
execute(-4.0) -> -4.0 / 4.0
execute(100.0) -> 100 / 4.0
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(MaxAbsScalarTransformer)
.SinceVersion(1)
@ -426,11 +569,13 @@ void RegisterMaxAbsScalarFeaturizerVer1() {
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) {
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0);
} else {
fail_type_inference("input 1 is expected to have a accepted type");
fail_type_inference("input 1 is expected to have an accepted type");
}
if (hasInputShape(ctx, 1)) {
propagateShapeFromInputToOutput(ctx, 1, 0);
}
});
}
@ -440,7 +585,7 @@ void RegisterMinMaxScalarFeaturizerVer1() {
during training.
C++-style pseudo signature:
template <typeanem T> std::double_t(T const &value);
template <typeanem T> double(T const &value);
Examples:
Given the training data [1, 2, 3, 4, 5];
@ -450,7 +595,7 @@ void RegisterMinMaxScalarFeaturizerVer1() {
execute(2) = 2 / 4
execute(20) = 20 / 4
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(MinMaxScalarTransformer)
.SinceVersion(1)
@ -493,17 +638,17 @@ void RegisterMissingDummiesFeaturizerVer1() {
Returns 1 if the input is null, 0 if it is not.
C++-style pseudo signature:
std::int8_t execute(std::float_t const &value);
std::int8_t execute(std::double_t const &value);
template <typename T> std::int8_t execute(T const &value);
int8 execute(float const &value);
int8 execute(double const &value);
template <typename T> int8 execute(T const &value);
Examples:
1.0 -> 0
NaN -> 1
"foo" -> 0
std::optional<std::string>() -> 1
std::optional<std::string>("bar") -> 0
)DOC";
std::optional<string>() -> 1
std::optional<string>("bar") -> 0
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(MissingDummiesTransformer)
.SinceVersion(1)
@ -541,6 +686,80 @@ void RegisterMissingDummiesFeaturizerVer1() {
});
}
void RegisterOneHotEncoderFeaturizerVer1() {
static const char* doc = R"DOC(
Produces a one hot vector based on categories calculated during training.
C++-style pseudo signature:
template <typename T> OneHotVector execute(T const &value);
Examples:
Assuming the training data [10, 20, 30, 40]...
execute(10) -> [0, 1, 0, 0, 0]
execute(20) -> [0, 0, 1, 0, 0]
execute(30) -> [0, 0, 0, 1, 0]
execute(40) -> [0, 0, 0, 0, 1]
execute(200) -> [1, 0, 0, 0, 0]
execute(-1) -> [1, 0, 0, 0, 0]
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(OneHotEncoderTransformer)
.SinceVersion(1)
.SetDomain(kMSFeaturizersDomain)
.SetDoc(doc)
.Input(
0,
"State",
"State generated during training that is used for prediction",
"T0")
.Input(
1,
"Input",
"No information is available",
"InputT")
.Output(0, "NumElements", "No information available", "OutputT0")
.Output(1, "Value", "No information available", "OutputT1")
.Output(2, "Index", "No information available", "OutputT0")
.TypeConstraint(
"T0",
{"tensor(uint8)"},
"No information is available")
.TypeConstraint(
"InputT",
{"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"},
"No information is available")
.TypeConstraint(
"OutputT0",
{"tensor(uint64)"},
"No information is available")
.TypeConstraint(
"OutputT1",
{"tensor(uint8)"},
"No information is available")
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
const bool has_shape = hasInputShape(ctx, 1);
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 0);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 0);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, 1);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 1);
}
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT64, 2);
if(has_shape) {
propagateShapeFromInputToOutput(ctx, 1, 2);
}
}
);
}
void RegisterRobustScalarFeaturizerVer1() {
static const char* doc = R"DOC(
MinMaxScalarEstimator + centering?
@ -550,7 +769,7 @@ void RegisterRobustScalarFeaturizerVer1() {
Examples:
TODO
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(RobustScalarTransformer)
.SinceVersion(1)
@ -599,11 +818,13 @@ void RegisterRobustScalarFeaturizerVer1() {
input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) {
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0);
} else {
fail_type_inference("input 1 is expected to have a accepted type");
fail_type_inference("input 1 is expected to have an accepted type");
}
if (hasInputShape(ctx, 1)) {
propagateShapeFromInputToOutput(ctx, 1, 0);
}
});
}
@ -612,12 +833,12 @@ void RegisterStringFeaturizerVer1() {
Converts the input into a string representation based on the input's type.
C++-style pseudo signature:
template <typename T> std::string execute(T const &value);
template <typename T> string execute(T const &value);
Examples:
execute(1) -> "1"
execute(3.14) -> "3.14"
)DOC";
)DOC";
MS_FEATURIZERS_OPERATOR_SCHEMA(StringTransformer)
.SinceVersion(1)

2
onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc
View file

@ -27,7 +27,7 @@ inline nonstd::optional<std::string> PreprocessOptional(std::string value) {
return value.empty() ? nonstd::optional<std::string>() : nonstd::optional<std::string>(std::move(value));
}
template<typename T>
template <typename T>
struct CatImputerTransformerImpl {
void operator()(OpKernelContext* ctx) const {
// Create the transformer

7
onnxruntime/featurizers_ops/cpu/date_time_transformer.cc
View file

@ -3,6 +3,7 @@
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/data_types_internal.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/DateTimeFeaturizer.h"
@ -29,7 +30,7 @@ class DateTimeTransformer final : public OpKernel {
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const std::int64_t* input_data(input_tensor->Data<std::int64_t>());
const int64_t* input_data(input_tensor->Data<int64_t>());
// Prepare the output
Tensor* year_tensor(ctx->Output(0, input_tensor->Shape()));
@ -80,7 +81,7 @@ class DateTimeTransformer final : public OpKernel {
const int64_t length(input_tensor->Shape().Size());
for (int64_t i = 0; i < length; ++i) {
auto result(transformer.execute(input_data[i]));
auto result(transformer.execute(std::chrono::system_clock::from_time_t(input_data[i])));
year_data[i] = std::move(result.year);
month_data[i] = std::move(result.month);
@ -116,7 +117,7 @@ ONNX_OPERATOR_KERNEL_EX(
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("T1", DataTypeImpl::GetTensorType<uint64_t>()),
.TypeConstraint("T1", DataTypeImpl::GetTensorType<int64_t>()),
DateTimeTransformer);
} // namespace featurizers

90
onnxruntime/featurizers_ops/cpu/hash_one_hot_vectorizer_transformer.cc Normal file
View file

@ -0,0 +1,90 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/data_types_internal.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/HashOneHotVectorizerFeaturizer.h"
#include "Archive.h"
namespace onnxruntime {
namespace featurizers {
template <typename InputT>
struct HashOneHotVectorizerTransformerImpl {
void operator()(OpKernelContext* ctx) const {
// Create the transformer
Microsoft::Featurizer::Featurizers::HashOneHotVectorizerTransformer<InputT> transformer(
[ctx](void) {
const auto* state_tensor(ctx->Input<Tensor>(0));
const uint8_t* const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return Microsoft::Featurizer::Featurizers::HashOneHotVectorizerTransformer<InputT>(archive);
}());
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const InputT* input_data(input_tensor->Data<InputT>());
// Prepare the output
Tensor* NumElements_tensor(ctx->Output(0, input_tensor->Shape()));
Tensor* Value_tensor(ctx->Output(1, input_tensor->Shape()));
Tensor* Index_tensor(ctx->Output(2, input_tensor->Shape()));
uint64_t* NumElements_data(NumElements_tensor->MutableData<uint64_t>());
uint8_t* Value_data(Value_tensor->MutableData<uint8_t>());
uint64_t* Index_data(Index_tensor->MutableData<uint64_t>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for (int64_t i = 0; i < length; ++i) {
auto result(transformer.execute(input_data[i]));
NumElements_data[i] = std::move(result.NumElements);
Value_data[i] = std::move(result.Value);
Index_data[i] = std::move(result.Index);
}
}
};
class HashOneHotVectorizerTransformer final : public OpKernel {
public:
explicit HashOneHotVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* ctx) const override {
utils::MLTypeCallDispatcher<HashOneHotVectorizerTransformerImpl, int8_t, uint8_t, int16_t, uint16_t, int32_t,
uint32_t, int64_t, uint64_t, float, double, bool, std::string>
t_disp(ctx->Input<Tensor>(1)->GetElementType());
t_disp.Invoke(ctx);
return Status::OK();
}
};
ONNX_OPERATOR_KERNEL_EX(
HashOneHotVectorizerTransformer,
kMSFeaturizersDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("InputT", {DataTypeImpl::GetTensorType<int8_t>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int16_t>(),
DataTypeImpl::GetTensorType<uint16_t>(),
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>(),
DataTypeImpl::GetTensorType<int64_t>(),
DataTypeImpl::GetTensorType<uint64_t>(),
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<bool>(),
DataTypeImpl::GetTensorType<std::string>()}),
HashOneHotVectorizerTransformer);
} // namespace featurizers
} // namespace onnxruntime

4
onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc
View file

@ -54,8 +54,7 @@ class ImputationMarkerTransformer final : public OpKernel {
}
Status Compute(OpKernelContext* ctx) const override {
utils::MLTypeCallDispatcher<ImputationMarkerTransformerImpl, float, double, std::string>
t_disp(ctx->Input<Tensor>(1)->GetElementType());
utils::MLTypeCallDispatcher<ImputationMarkerTransformerImpl, float, double, std::string> t_disp(ctx->Input<Tensor>(1)->GetElementType());
t_disp.Invoke(ctx);
return Status::OK();
}
@ -72,5 +71,6 @@ ONNX_OPERATOR_KERNEL_EX(
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<std::string>()}),
ImputationMarkerTransformer);
} // namespace featurizers
} // namespace onnxruntime

2
onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc
View file

@ -31,7 +31,7 @@ struct LabelEncoderTransformerImpl {
// Prepare the output
Tensor* output_tensor(ctx->Output(0, input_tensor->Shape()));
std::uint32_t* output_data(output_tensor->MutableData<std::uint32_t>());
uint32_t* output_data(output_tensor->MutableData<uint32_t>());
// Execute
const int64_t length(input_tensor->Shape().Size());

3
onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc
View file

@ -72,7 +72,8 @@ class MaxAbsScalarTransformer final : public OpKernel {
Status Compute(OpKernelContext* ctx) const override {
utils::MLTypeCallDispatcher<MaxAbsScalarTransformerImpl, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t,
int64_t, uint64_t, float, double> t_disp(ctx->Input<Tensor>(1)->GetElementType());
int64_t, uint64_t, float, double>
t_disp(ctx->Input<Tensor>(1)->GetElementType());
t_disp.Invoke(ctx);
return Status::OK();
}

3
onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc
View file

@ -54,8 +54,7 @@ class MissingDummiesTransformer final : public OpKernel {
}
Status Compute(OpKernelContext* ctx) const override {
utils::MLTypeCallDispatcher<MissingDummiesTransformerImpl, float, double, std::string>
t_disp(ctx->Input<Tensor>(1)->GetElementType());
utils::MLTypeCallDispatcher<MissingDummiesTransformerImpl, float, double, std::string> t_disp(ctx->Input<Tensor>(1)->GetElementType());
t_disp.Invoke(ctx);
return Status::OK();
}

90
onnxruntime/featurizers_ops/cpu/one_hot_encoder_transformer.cc Normal file
View file

@ -0,0 +1,90 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/common.h"
#include "core/framework/data_types.h"
#include "core/framework/data_types_internal.h"
#include "core/framework/op_kernel.h"
#include "Featurizers/OneHotEncoderFeaturizer.h"
#include "Archive.h"
namespace onnxruntime {
namespace featurizers {
template <typename InputT>
struct OneHotEncoderTransformerImpl {
void operator()(OpKernelContext* ctx) const {
// Create the transformer
Microsoft::Featurizer::Featurizers::OneHotEncoderTransformer<InputT> transformer(
[ctx](void) {
const auto* state_tensor(ctx->Input<Tensor>(0));
const uint8_t* const state_data(state_tensor->Data<uint8_t>());
Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
return Microsoft::Featurizer::Featurizers::OneHotEncoderTransformer<InputT>(archive);
}());
// Get the input
const auto* input_tensor(ctx->Input<Tensor>(1));
const InputT* input_data(input_tensor->Data<InputT>());
// Prepare the output
Tensor* NumElements_tensor(ctx->Output(0, input_tensor->Shape()));
Tensor* Value_tensor(ctx->Output(1, input_tensor->Shape()));
Tensor* Index_tensor(ctx->Output(2, input_tensor->Shape()));
uint64_t* NumElements_data(NumElements_tensor->MutableData<uint64_t>());
uint8_t* Value_data(Value_tensor->MutableData<uint8_t>());
uint64_t* Index_data(Index_tensor->MutableData<uint64_t>());
// Execute
const int64_t length(input_tensor->Shape().Size());
for (int64_t i = 0; i < length; ++i) {
auto result(transformer.execute(input_data[i]));
NumElements_data[i] = std::move(result.NumElements);
Value_data[i] = std::move(result.Value);
Index_data[i] = std::move(result.Index);
}
}
};
class OneHotEncoderTransformer final : public OpKernel {
public:
explicit OneHotEncoderTransformer(const OpKernelInfo& info) : OpKernel(info) {
}
Status Compute(OpKernelContext* ctx) const override {
utils::MLTypeCallDispatcher<OneHotEncoderTransformerImpl, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t,
int64_t, uint64_t, float, double, bool, std::string>
t_disp(ctx->Input<Tensor>(1)->GetElementType());
t_disp.Invoke(ctx);
return Status::OK();
}
};
ONNX_OPERATOR_KERNEL_EX(
OneHotEncoderTransformer,
kMSFeaturizersDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("InputT", {DataTypeImpl::GetTensorType<int8_t>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int16_t>(),
DataTypeImpl::GetTensorType<uint16_t>(),
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>(),
DataTypeImpl::GetTensorType<int64_t>(),
DataTypeImpl::GetTensorType<uint64_t>(),
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<bool>(),
DataTypeImpl::GetTensorType<std::string>()}),
OneHotEncoderTransformer);
} // namespace featurizers
} // namespace onnxruntime

4
onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc
View file

@ -12,11 +12,13 @@ namespace featurizers {
// Forward declarations
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, ImputationMarkerTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, LabelEncoderTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MaxAbsScalarTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MinMaxScalarTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MissingDummiesTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, OneHotEncoderTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, RobustScalarTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer);
@ -25,11 +27,13 @@ Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, ImputationMarkerTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, LabelEncoderTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MaxAbsScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MinMaxScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MissingDummiesTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, OneHotEncoderTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, RobustScalarTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer)>,

37
onnxruntime/test/featurizers_ops/datetimetransformer_test.cc
View file

@ -4,6 +4,7 @@
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Archive.h"
#include "Featurizers/DateTimeFeaturizer.h"
namespace dft = Microsoft::Featurizer::Featurizers;
@ -13,12 +14,21 @@ using SysClock = std::chrono::system_clock;
namespace onnxruntime {
namespace test {
std::vector<uint8_t> GetStream () {
dft::DateTimeTransformer dt("", "");
Microsoft::Featurizer::Archive ar;
dt.save(ar);
return ar.commit();
}
TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_04) {
const time_t date = 217081624;
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0});
auto stream = GetStream();
auto dim = static_cast<int64_t>(stream.size());
test.AddInput<uint8_t>("State", {dim}, stream);
// We are adding a scalar Tensor in this instance
test.AddInput<int64_t>("Date", {1}, {date});
@ -76,17 +86,20 @@ TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_04) {
TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_05) {
const time_t date = 217081625;
const auto date_tp = SysClock::from_time_t(date);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0});
auto stream = GetStream();
auto dim = static_cast<int64_t>(stream.size());
test.AddInput<uint8_t>("State", {dim}, stream);
// We are adding a scalar Tensor in this instance
test.AddInput<int64_t>("Date", {1}, {date});
dft::DateTimeTransformer dt("", "");
dft::TimePoint tp(dt.execute(date));
dft::TimePoint tp(dt.execute(date_tp));
ASSERT_EQ(tp.year, 1976);
ASSERT_EQ(tp.month, dft::TimePoint::NOVEMBER);
ASSERT_EQ(tp.day, 17);
@ -137,19 +150,24 @@ TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_05) {
TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17__12_27_05_and_past_1976_nov_17_12_27_04) {
const time_t date1 = 217081625;
const auto date1_tp = SysClock::from_time_t(date1);
const time_t date2 = 217081624;
const auto date2_tp = SysClock::from_time_t(date2);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0});
auto stream = GetStream();
auto dim = static_cast<int64_t>(stream.size());
test.AddInput<uint8_t>("State", {dim}, stream);
// We are adding a scalar Tensor in this instance
test.AddInput<int64_t>("Date", {2}, {date1, date2});
dft::DateTimeTransformer dt("", "");
dft::TimePoint tp1(dt.execute(date1));
dft::TimePoint tp2(dt.execute(date2));
dft::TimePoint tp1(dt.execute(date1_tp));
dft::TimePoint tp2(dt.execute(date2_tp));
// Date1
ASSERT_EQ(tp1.year, 1976);
@ -225,17 +243,20 @@ TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17__12_27_05_and_past_1
TEST(FeaturizersTests, DateTimeTransformer_future_2025_june_30) {
const time_t date = 1751241600;
const auto date_tp = std::chrono::system_clock::from_time_t(date);
OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain);
// Add state input
test.AddInput<uint8_t>("State", {8}, {1, 0, 0, 0, 0, 0, 0, 0});
auto stream = GetStream();
auto dim = static_cast<int64_t>(stream.size());
test.AddInput<uint8_t>("State", {dim}, stream);
// We are adding a scalar Tensor in this instance
test.AddInput<int64_t>("Date", {1}, {date});
dft::DateTimeTransformer dt("", "");
dft::TimePoint tp = dt.execute(date);
dft::TimePoint tp = dt.execute(date_tp);
ASSERT_EQ(tp.year, 2025);
ASSERT_EQ(tp.month, dft::TimePoint::JUNE);
ASSERT_EQ(tp.day, 30);

89
onnxruntime/test/featurizers_ops/hash_one_hot_encoder_transformer_test.cc Normal file
View file

@ -0,0 +1,89 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Featurizers/HashOneHotVectorizerFeaturizer.h"
#include "Featurizers/TestHelpers.h"
#include "Archive.h"
namespace NS = Microsoft::Featurizer;
namespace onnxruntime {
namespace test {
template <typename T>
std::vector<uint8_t> GetStream() {
NS::Featurizers::HashOneHotVectorizerTransformer<T> hvtransformer(2, 100);
NS::Archive ar;
hvtransformer.save(ar);
return ar.commit();
}
TEST(FeaturizersTests, HashOneHotVectorizerTransformer_int8) {
using Type = int8_t;
auto stream = GetStream<Type>();
auto dim = static_cast<int64_t>(stream.size());
OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<Type>("Input", {1}, {15});
test.AddOutput<uint64_t>("NumElements", {1}, {100u});
test.AddOutput<uint8_t>("Value", {1}, {1u});
test.AddOutput<uint64_t>("Index", {1}, {29u});
test.Run();
}
TEST(FeaturizersTests, HashOneHotVectorizerTransformer_int32) {
using Type = int32_t;
auto stream = GetStream<Type>();
auto dim = static_cast<int64_t>(stream.size());
OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<Type>("Input", {1}, {15});
test.AddOutput<uint64_t>("NumElements", {1}, {100u});
test.AddOutput<uint8_t>("Value", {1}, {1u});
test.AddOutput<uint64_t>("Index", {1}, {22u});
test.Run();
}
TEST(FeaturizersTests, HashOneHotVectorizerTransformer_double) {
using Type = double;
auto stream = GetStream<Type>();
auto dim = static_cast<int64_t>(stream.size());
OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<Type>("Input", {1}, {15.0});
test.AddOutput<uint64_t>("NumElements", {1}, {100u});
test.AddOutput<uint8_t>("Value", {1}, {1u});
test.AddOutput<uint64_t>("Index", {1}, {99u});
test.Run();
}
TEST(FeaturizersTests, HashOneHotVectorizerTransformer_string) {
using Type = std::string;
auto stream = GetStream<Type>();
auto dim = static_cast<int64_t>(stream.size());
OpTester test("HashOneHotVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<Type>("Input", {1}, {"hello"});
test.AddOutput<uint64_t>("NumElements", {1}, {100u});
test.AddOutput<uint8_t>("Value", {1}, {1u});
test.AddOutput<uint64_t>("Index", {1}, {25u});
test.Run();
}
} // namespace test
} // namespace onnxruntime

119
onnxruntime/test/featurizers_ops/one_hot_encoder_test.cc Normal file
View file

@ -0,0 +1,119 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "Featurizers/OneHotEncoderFeaturizer.h"
#include "Featurizers/TestHelpers.h"
#include "Archive.h"
namespace NS = Microsoft::Featurizer;
namespace onnxruntime {
namespace test {
template <typename InputType>
std::vector<uint8_t> GetStream(const std::vector<std::vector<InputType>>& trainingBatches, bool allowMissingValues) {
using Estimator = NS::Featurizers::OneHotEncoderEstimator<InputType>;
Estimator estimator(NS::CreateTestAnnotationMapsPtr(1), 0, allowMissingValues);
NS::TestHelpers::Train<Estimator, InputType>(estimator, trainingBatches);
typename Estimator::TransformerUniquePtr pTransformer(estimator.create_transformer());
NS::Archive ar;
pTransformer->save(ar);
return ar.commit();
}
TEST(FeaturizersTests, OneHotEncoder_uint32_t) {
using InputType = uint32_t;
auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
NS::TestHelpers::make_vector<InputType>(10, 20, 10),
NS::TestHelpers::make_vector<InputType>(30),
NS::TestHelpers::make_vector<InputType>(10, 10, 11, 15),
NS::TestHelpers::make_vector<InputType>(18, 8));
auto stream = GetStream<InputType>(trainingBatches, false);
auto dim = static_cast<int64_t>(stream.size());
OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<InputType>("Input", {5}, {11u, 8u, 10u, 15u, 20u});
test.AddOutput<uint64_t>("NumElements", {5}, {7u, 7u, 7u, 7u, 7u});
test.AddOutput<uint8_t>("Value", {5}, {1u, 1u, 1u, 1u, 1u});
test.AddOutput<uint64_t>("Index", {5}, {2u, 0u, 1u, 3u, 5u});
test.Run();
}
TEST(FeaturizersTests, OneHotEncoder_string) {
using InputType = std::string;
auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
NS::TestHelpers::make_vector<InputType>("orange", "apple", "orange",
"grape", "carrot", "carrot",
"peach", "banana", "orange"));
auto stream = GetStream<InputType>(trainingBatches, false);
auto dim = static_cast<int64_t>(stream.size());
OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<InputType>("Input", {4}, {"banana", "grape", "apple", "orange"});
test.AddOutput<uint64_t>("NumElements", {4}, {6u, 6u, 6u, 6u});
test.AddOutput<uint8_t>("Value", {4}, {1u, 1u, 1u, 1u});
test.AddOutput<uint64_t>("Index", {4}, {1u, 3u, 0u, 4u});
test.Run();
}
TEST(FeaturizersTests, OneHotEncoder_unseen_values) {
using InputType = std::string;
auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
NS::TestHelpers::make_vector<InputType>("orange", "apple", "orange",
"grape", "carrot", "carrot",
"peach", "banana", "orange"));
auto stream = GetStream<InputType>(trainingBatches, true);
auto dim = static_cast<int64_t>(stream.size());
OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<InputType>("Input", {5}, {"banana", "grape", "apple", "orange", "hello"});
test.AddOutput<uint64_t>("NumElements", {5}, {7u, 7u, 7u, 7u, 7u});
test.AddOutput<uint8_t>("Value", {5}, {1u, 1u, 1u, 1u, 1u});
test.AddOutput<uint64_t>("Index", {5}, {2u, 4u, 1u, 5u, 0u});
test.Run();
}
TEST(FeaturizersTests, OneHotEncoder_unseen_values_throws) {
using InputType = std::string;
auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
NS::TestHelpers::make_vector<InputType>("orange", "apple", "orange",
"grape", "carrot", "carrot",
"peach", "banana", "orange"));
auto stream = GetStream<InputType>(trainingBatches, false);
auto dim = static_cast<int64_t>(stream.size());
OpTester test("OneHotEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain);
test.AddInput<uint8_t>("State", {dim}, stream);
test.AddInput<InputType>("Input", {5}, {"banana", "grape", "apple", "orange", "hello"});
test.AddOutput<uint64_t>("NumElements", {5}, {7u, 7u, 7u, 7u, 7u});
test.AddOutput<uint8_t>("Value", {5}, {1u, 1u, 1u, 1u, 1u});
test.AddOutput<uint64_t>("Index", {5}, {2u, 4u, 1u, 5u, 0u});
test.Run(OpTester::ExpectResult::kExpectFailure);
}
} // namespace test
} // namespace onnxruntime