saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

FeaturizersLibrary update and add variadic Input/Output to TimeSeriesImputer (#3674)

Browse source
This commit is contained in:
Ye Wang 2020-04-24 08:53:00 -07:00 committed by GitHub
parent 6d4f2f5bf9
commit 5c7f616431
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 156 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cgmanifest.json
View file

@ -450,7 +450,7 @@
{
"component": {
"git": {
"commitHash": "f01d8beeac92b54bf93df8053a2acb6a8a10f03c",
"commitHash": "fd5fe3de507d4a19f5923c5d4c267e3d730500a9",
"repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git"
},
"type": "git"

2
cmake/external/FeaturizersLibrary vendored

@ -1 +1 @@
Subproject commit f01d8beeac92b54bf93df8053a2acb6a8a10f03c
Subproject commit fd5fe3de507d4a19f5923c5d4c267e3d730500a9

41
onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc
View file

@ -2066,11 +2066,20 @@ void RegisterTimeSeriesImputerFeaturizerVer1() {
"Keys",
"Composite keys tensor of shape [R][K]. R is the same as Input(1)",
"T2")
// The first input in Input(3)(a variadic tensor that has multiple output) is the below commented input
// for the ONNX requirement that allow 0 size of variadic input
// .Input(
// 3,
// "Data",
// "It is a data tensor of shape [R][C] where R - rows and C - columns. R must be the same with Input(1)",
// "T2")
.Input(
3,
"Data",
"It is a data tensor of shape [R][C] where R - rows and C - columns. R must be the same with Input(1)",
"T2")
"Input",
"Variadic number of Inputs containing tensors of different type",
"T",
ONNX_NAMESPACE::OpSchema::FormalParameterOption::Variadic,
false)
.Output(
0,
"Added",
@ -2086,12 +2095,21 @@ void RegisterTimeSeriesImputerFeaturizerVer1() {
"ImputedKeys",
"Contains keys along with the imputed keys. Tensor of shape [IR][K].",
"T2")
// The first output in Output(3)(a variadic tensor that has multiple output) is the below commented output
// for the ONNX requirement that allow 0 size of variadic output
// .Output(
// 3,
// "ImputedData",
// "Tensor of shape [IR][C] where IR is the number of rows in the output."
// "C is the number of columns.",
// "T2")
.Output(
3,
"ImputedData",
"Tensor of shape [IR][C] where IR is the number of rows in the output."
"C is the number of columns.",
"T2")
"Output",
"Variadic number of Outputs containing tensors of different type",
"T",
ONNX_NAMESPACE::OpSchema::FormalParameterOption::Variadic,
false)
.TypeConstraint(
"T0",
{"tensor(uint8)"},
@ -2108,6 +2126,11 @@ void RegisterTimeSeriesImputerFeaturizerVer1() {
"T3",
{"tensor(bool)"},
"Boolean Tensor")
.TypeConstraint(
"T",
{"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)",
"tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"},
"No information is available")
.TypeAndShapeInferenceFunction(
[](ONNX_NAMESPACE::InferenceContext& ctx) {
propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_BOOL, 0);
@ -2132,12 +2155,12 @@ void RegisterTimeSeriesImputerFeaturizerVer1() {
ONNX_NAMESPACE::updateOutputShape(ctx, 2, shape);
}
// Data shape
//Data Shape & Variadic I/O shapes
propagateElemTypeFromInputToOutput(ctx, 3, 3);
if (hasInputShape(ctx, 3)) {
const auto& input3_shape = getInputShape(ctx, 3);
if (input3_shape.dim_size() != 2) {
fail_shape_inference("Expecting data to have 2 dimensions");
fail_shape_inference("Expecting data and variadic inputs to have 2 dimensions");
}
ONNX_NAMESPACE::TensorShapeProto shape;
shape.add_dim();

91
onnxruntime/featurizers_ops/cpu/time_series_imputer_transformer.cc
View file

@ -116,6 +116,61 @@ struct FromStringOptional<std::string> {
};
} // namespace timeseries_imputer_details
template<typename T>
struct Helper{
static T GetDefaultValue() {
return static_cast<T>(0);
}
};
template<>
struct Helper<std::string>{
static std::string GetDefaultValue() {
return "";
}
};
template<>
struct Helper<float>{
static float GetDefaultValue() {
return std::numeric_limits<float>::quiet_NaN();
}
};
template<>
struct Helper<double> {
static double GetDefaultValue() {
return std::numeric_limits<double>::quiet_NaN();
}
};
template<>
struct Helper<bool> {
static bool GetDefaultValue() {
return false;
}
};
template <typename T>
struct GenerateImputedColumnsImpl {
void operator()(const Tensor* variadic_input_tensor, Tensor* output_after_impute_tensor,
const std::vector<bool>& is_row_imputed, int64_t input_row_size) const {
const T* input_data(variadic_input_tensor->template Data<T>());
T* output_after_impute_data = output_after_impute_tensor->MutableData<T>();
for (int row_idx = 0; row_idx < static_cast<int>(is_row_imputed.size()); ++row_idx) {
if (!is_row_imputed[row_idx]) {
output_after_impute_data = std::copy(input_data, input_data + input_row_size, output_after_impute_data);
input_data += input_row_size;
} else {
for (int col_idx = 0; col_idx < input_row_size; col_idx++) {
*output_after_impute_data++ = Helper<T>::GetDefaultValue();
}
}
}
}
};
template <typename T>
struct TimeSeriesImputerTransformerImpl {
void operator()(OpKernelContext* ctx, int64_t rows) {
@ -180,7 +235,10 @@ struct TimeSeriesImputerTransformerImpl {
auto* keys_output = ctx->Output(2, keys_shape)->template MutableData<T>();
auto* data_output = ctx->Output(3, data_shape)->template MutableData<T>();
std::vector<bool> is_row_imputed;
is_row_imputed.reserve(output_rows_num);
for (const auto& out : output_rows) {
is_row_imputed.push_back(std::get<0>(out));
*added_output++ = std::get<0>(out);
*time_output++ = ToSecs(std::get<1>(out));
const auto& imputed_keys = std::get<2>(out);
@ -194,6 +252,25 @@ struct TimeSeriesImputerTransformerImpl {
data_output = std::transform(imputed_data.cbegin(), imputed_data.cend(), data_output,
FromStringOptional<T>());
}
const int variadic_input_start_id = ctx->NumVariadicInputs(0) + ctx->NumVariadicInputs(1) + ctx->NumVariadicInputs(2) + 1;
const int variadic_input_end_id = variadic_input_start_id + ctx->NumVariadicInputs(3) - 1;
for (int input_id = variadic_input_start_id; input_id < variadic_input_end_id; ++input_id) {
const auto* variadic_input_tensor(ctx->Input<Tensor>(input_id)); //2-d tensor
const int64_t input_row_size = variadic_input_tensor->Shape()[1];
TensorShape output_after_impute_shape({static_cast<int64_t>(is_row_imputed.size()), input_row_size});
Tensor* output_after_impute_tensor(ctx->Output(input_id, output_after_impute_shape));
const auto elem_type = variadic_input_tensor->GetElementType();
utils::MLTypeCallDispatcher<GenerateImputedColumnsImpl,
int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t,
float, double, bool, std::string> t_disp(elem_type);
t_disp.Invoke(variadic_input_tensor, output_after_impute_tensor, is_row_imputed, input_row_size);
}
}
};
@ -238,7 +315,19 @@ ONNX_OPERATOR_KERNEL_EX(
KernelDefBuilder()
.TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
.TypeConstraint("T1", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("T2", DataTypeImpl::GetTensorType<std::string>()),
.TypeConstraint("T2", DataTypeImpl::GetTensorType<std::string>())
.TypeConstraint("T", {DataTypeImpl::GetTensorType<int8_t>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int16_t>(),
DataTypeImpl::GetTensorType<uint16_t>(),
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>(),
DataTypeImpl::GetTensorType<int64_t>(),
DataTypeImpl::GetTensorType<uint64_t>(),
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<bool>(),
DataTypeImpl::GetTensorType<std::string>()}),
TimeSeriesImputerTransformer);
} // namespace featurizers
} // namespace onnxruntime

33
onnxruntime/test/featurizers_ops/time_series_imputer_transformer_test.cc
View file

@ -187,10 +187,41 @@ TEST(FeaturizersTests, RowImputation_2_grains_1_gap_input_interleaved) {
OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
AddInputs(test, {{tuple_0, tuple_5, tuple_1, tuple_6}}, {tuple_0, tuple_5_inf, tuple_2, tuple_7},
{NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward);
AddOutputs(test, {false, false, true, false, true, false}, {tp_0, tp_5, tp_1, tp_2, tp_6, tp_7},
{"a", "b", "a", "a", "b", "b"}, {"14.5", "18", "114.5", "118", "14.5", "18", "14.5", "12", "114.5", "118", "114.5", "112"});
test.Run();
}
TEST(FeaturizersTests, RowImputation_2_grains_1_gap_input_interleaved_add_additional_imputed_columns) {
std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
auto tp_0 = GetTimePoint(now, 0);
auto tp_1 = GetTimePoint(now, 1);
auto tp_2 = GetTimePoint(now, 2);
auto tp_5 = GetTimePoint(now, 5);
auto tp_6 = GetTimePoint(now, 6);
auto tp_7 = GetTimePoint(now, 7);
auto tuple_0 = std::make_tuple(tp_0, std::vector<std::string>{"a"}, std::vector<nonstd::optional<std::string>>{"14.5", "18"});
auto tuple_2 = std::make_tuple(GetTimePoint(now, 2), std::vector<std::string>{"a"}, std::vector<nonstd::optional<std::string>>{nonstd::optional<std::string>{}, "12"});
auto tuple_5 = std::make_tuple(tp_5, std::vector<std::string>{"b"}, std::vector<nonstd::optional<std::string>>{"14.5", "18"});
auto tuple_5_inf = std::make_tuple(tp_5, std::vector<std::string>{"b"}, std::vector<nonstd::optional<std::string>>{"114.5", "118"});
auto tuple_1 = std::make_tuple(tp_1, std::vector<std::string>{"a"}, std::vector<nonstd::optional<std::string>>{nonstd::optional<std::string>{}, "12"});
auto tuple_6 = std::make_tuple(tp_6, std::vector<std::string>{"b"}, std::vector<nonstd::optional<std::string>>{nonstd::optional<std::string>{}, "12"});
auto tuple_7 = std::make_tuple(GetTimePoint(now, 7), std::vector<std::string>{"b"}, std::vector<nonstd::optional<std::string>>{nonstd::optional<std::string>{}, "112"});
OpTester test("TimeSeriesImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
AddInputs(test, {{tuple_0, tuple_5, tuple_1, tuple_6}}, {tuple_0, tuple_5_inf, tuple_2, tuple_7},
{NS::TypeId::Float64, NS::TypeId::Float64}, false, NS::Featurizers::Components::TimeSeriesImputeStrategy::Forward);
test.AddInput<int64_t>("Input_1", {4, 1}, {1, 2, 3, 4});
test.AddInput<float>("Input_2", {4, 1}, {1, 2, 3, 4});
test.AddInput<std::string>("Input_3", {4, 1}, {"1", "2", "3", "4"});
test.AddInput<bool>("Input_4", {4, 1}, {false, true, true, true});
AddOutputs(test, {false, false, true, false, true, false}, {tp_0, tp_5, tp_1, tp_2, tp_6, tp_7},
{"a", "b", "a", "a", "b", "b"}, {"14.5", "18", "114.5", "118", "14.5", "18", "14.5", "12", "114.5", "118", "114.5", "112"});
test.AddOutput<int64_t>("Output_1", {6, 1}, {1, 2, 0, 3, 0, 4});
test.AddOutput<float>("Output_2", {6, 1}, {1, 2, std::numeric_limits<float>::quiet_NaN(), 3, std::numeric_limits<float>::quiet_NaN(), 4});
test.AddOutput<std::string>("Output_3", {6, 1}, {"1", "2", "", "3", "", "4"});
test.AddOutput<bool>("Output_4", {6, 1}, {false, true, false, true, false, true});
test.Run();
}