From 88d48063fae7c29156fcfb6e035294920e0a2e19 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 10 Feb 2021 10:01:16 -0800 Subject: [PATCH] Log warning when GetGradientForOp() silently fails. (#6586) * Add warning when GetGradientForOp() silently fails. In some cases, `GetGradientForOp()` can return without creating any nodes, which may lead to an invalid graph being created. --- .../orttraining/core/framework/gradient_graph_builder.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/orttraining/orttraining/core/framework/gradient_graph_builder.cc b/orttraining/orttraining/core/framework/gradient_graph_builder.cc index 6fa618316c..08811af545 100644 --- a/orttraining/orttraining/core/framework/gradient_graph_builder.cc +++ b/orttraining/orttraining/core/framework/gradient_graph_builder.cc @@ -211,7 +211,11 @@ Status GradientGraphBuilder::Build(const std::unordered_set* p_init } GradientDef node_defs = GetGradientForOp(gradient_graph_config_, graph_, node, output_args_need_grad, input_args_need_grad, logger_); - + if (node_defs.empty()) { + LOGS(logger_, WARNING) << "GetGradientForOp() did not create any nodes for node " + << node->Name() << " of type " << node->OpType() << "."; + } + // updates arg name if gradient accumulation is needed for (auto& op_def : node_defs) { for (auto& arg : op_def.output_args) {