Disable PERF* rules in ruff to allow better readability (#16834)

### Description

Disable two PERF* rules in ruff to allow better readability. Rational
commented inline. This change also removes the unused noqa directives
because of the rule change.

### Motivation and Context

Readability
This commit is contained in:
Justin Chu 2023-07-25 15:38:22 -07:00 committed by GitHub
parent 53c771f215
commit 0c1a5098dc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
68 changed files with 133 additions and 135 deletions

View file

@ -683,7 +683,7 @@ def get_onnx_example(op_name):
try:
mod = importlib.import_module(m)
module = m
except ImportError: # noqa: PERF203
except ImportError:
continue
if module is None:
# Unable to find an example for 'op_name'.

View file

@ -86,7 +86,7 @@ for x in [
try:
r = sess.run([output_name], {input_name: x})
print(f"Shape={x.shape} and predicted labels={r}")
except (RuntimeError, InvalidArgument) as e: # noqa: PERF203
except (RuntimeError, InvalidArgument) as e:
print(f"ERROR with Shape={x.shape} - {e}")
for x in [
@ -99,7 +99,7 @@ for x in [
try:
r = sess.run(None, {input_name: x})
print(f"Shape={x.shape} and predicted probabilities={r[1]}")
except (RuntimeError, InvalidArgument) as e: # noqa: PERF203
except (RuntimeError, InvalidArgument) as e:
print(f"ERROR with Shape={x.shape} - {e}")
#########################
@ -114,5 +114,5 @@ for x in [
try:
r = sess.run([output_name], {input_name: x})
print(f"Shape={x.shape} and predicted labels={r}")
except (RuntimeError, InvalidArgument) as e: # noqa: PERF203
except (RuntimeError, InvalidArgument) as e:
print(f"ERROR with Shape={x.shape} - {e}")

View file

@ -66,7 +66,7 @@ class OnnxRuntimeBackend(Backend):
" Got Domain '{}' version '{}'.".format(domain, opset.version)
)
return False, error_message
except AttributeError: # noqa: PERF203
except AttributeError:
# for some CI pipelines accessing helper.OP_SET_ID_VERSION_MAP
# is generating attribute error. TODO investigate the pipelines to
# fix this error. Falling back to a simple version check when this error is encountered

View file

@ -191,7 +191,7 @@ class Session:
missing_input_names = []
for input in self._inputs_meta:
if input.name not in feed_input_names and not input.type.startswith("optional"):
missing_input_names.append(input.name) # noqa: PERF401
missing_input_names.append(input.name)
if missing_input_names:
raise ValueError(
f"Required inputs ({missing_input_names}) are missing from input feed ({feed_input_names})."

View file

@ -96,7 +96,7 @@ class ParseIOInfoAction(argparse.Action):
try:
comp_strs = io_str.split(";")
except ValueError: # noqa: PERF203
except ValueError:
parser.error(f"{opt_str}: {io_meta_name} info must be separated by ';'")
if len(comp_strs) != 3:

View file

@ -78,7 +78,7 @@ def _test_batched_gemm(
for i in range(batch):
try:
np.testing.assert_allclose(my_cs[i], ref_cs[i], rtol=bounds[i])
except Exception as err: # noqa: PERF203
except Exception as err:
header = "*" * 30 + impl + "*" * 30
print(header, bounds[i])
print(err)

View file

@ -182,7 +182,7 @@ def _test_gemm_softmax_gemm_permute(
is_zero_tol, atol, rtol = 1e-3, 2e-2, 1e-2
not_close_to_zeros = np.abs(ref) > is_zero_tol
np.testing.assert_allclose(out[not_close_to_zeros], ref[not_close_to_zeros], atol=atol, rtol=rtol)
except Exception as err: # noqa: PERF203
except Exception as err:
header = "*" * 30 + impl + "*" * 30
print(header)
print(err)

View file

@ -58,7 +58,7 @@ def _test_gemm(func, dtype: str, transa: bool, transb: bool, m: int, n: int, k:
try:
np.testing.assert_allclose(my_c, ref_c, rtol=bound)
except Exception as err: # noqa: PERF203
except Exception as err:
header = "*" * 30 + impl + "*" * 30
print(header)
print(err)

View file

@ -82,7 +82,7 @@ def _test_strided_batched_gemm(
for i in range(batch):
try:
np.testing.assert_allclose(my_c[i], ref_c[i], rtol=bounds[i])
except Exception as err: # noqa: PERF203
except Exception as err:
header = "*" * 30 + impl + "*" * 30
print(header, bounds[i])
print(err)

View file

@ -96,7 +96,7 @@ def unregister():
for name in _registered_ops:
try:
torch.onnx.unregister_custom_op_symbolic(name, _OPSET_VERSION)
except AttributeError: # noqa: PERF203
except AttributeError:
# The symbolic_registry module was removed in PyTorch 1.13.
# We are importing it here for backwards compatibility
# because unregister_custom_op_symbolic is not available before PyTorch 1.12

View file

@ -370,7 +370,7 @@ class HistogramCalibrater(CalibraterBase):
self.tensors_to_calibrate, value_infos = self.select_tensors_to_calibrate(self.model)
for tensor in self.tensors_to_calibrate:
if tensor not in self.model_original_outputs:
self.model.graph.output.append(value_infos[tensor]) # noqa: PERF401
self.model.graph.output.append(value_infos[tensor])
onnx.save(
self.model,

View file

@ -186,7 +186,7 @@ class ONNXModel:
parents = []
for input in node.input:
if input in output_name_to_node:
parents.append(output_name_to_node[input]) # noqa: PERF401
parents.append(output_name_to_node[input])
return parents
def get_parent(self, node, idx, output_name_to_node=None):
@ -222,7 +222,7 @@ class ONNXModel:
for node in graph.node:
for node_input in node.input:
if node_input == initializer.name:
nodes.append(node) # noqa: PERF401
nodes.append(node)
return nodes
@staticmethod
@ -379,7 +379,7 @@ class ONNXModel:
and not self.is_graph_output(node.output[0])
and node.output[0] not in input_name_to_nodes
):
unused_nodes.append(node) # noqa: PERF401
unused_nodes.append(node)
self.remove_nodes(unused_nodes)

View file

@ -145,7 +145,7 @@ def collect_activations(
intermediate_outputs = []
for input_d in input_reader:
intermediate_outputs.append(inference_session.run(None, input_d)) # noqa: PERF401
intermediate_outputs.append(inference_session.run(None, input_d))
if not intermediate_outputs:
raise RuntimeError("No data is collected while running augmented model!")

View file

@ -1008,13 +1008,13 @@ class SymbolicShapeInference:
right_ellipsis_index = right_equation.find(b"...")
if right_ellipsis_index != -1:
for i in range(num_ellipsis_indices):
new_sympy_shape.append(shape[i]) # noqa: PERF401
new_sympy_shape.append(shape[i])
for c in right_equation:
if c != 46: # c != b'.'
new_sympy_shape.append(letter_to_dim[c]) # noqa: PERF401
new_sympy_shape.append(letter_to_dim[c])
else:
for i in range(num_ellipsis_indices):
new_sympy_shape.append(shape[i]) # noqa: PERF401
new_sympy_shape.append(shape[i])
for c in left_equation:
if c != 44 and c != 46: # c != b',' and c != b'.':
if c in num_letter_occurrences:

View file

@ -138,7 +138,7 @@ def run_trt_standalone(trtexec, model_name, model_path, test_data_dir, all_input
logger.info(loaded_input)
shape = []
for j in all_inputs_shape[i]:
shape.append(str(j)) # noqa: PERF401
shape.append(str(j))
shape = "x".join(shape)
shape = name + ":" + shape
input_shape.append(shape)
@ -266,7 +266,7 @@ def get_ort_session_inputs_and_outputs(name, session, ort_input):
for i in range(len(session.get_inputs())):
sess_inputs[session.get_inputs()[i].name] = ort_input[i]
for i in range(len(session.get_outputs())):
sess_outputs.append(session.get_outputs()[i].name) # noqa: PERF401
sess_outputs.append(session.get_outputs()[i].name)
return (sess_inputs, sess_outputs)
@ -406,7 +406,7 @@ def inference_ort(
runtime = runtime[1:] # remove warmup
runtimes += runtime
except Exception as e: # noqa: PERF203
except Exception as e:
logger.error(e)
if track_memory:
end_memory_tracking(p, success)
@ -605,7 +605,7 @@ def validate(all_ref_outputs, all_outputs, rtol, atol, percent_mismatch):
# abs(desired-actual) < rtol * abs(desired) + atol
try:
np.testing.assert_allclose(ref_o, o, rtol, atol)
except Exception as e: # noqa: PERF203
except Exception as e:
if percentage_in_allowed_threshold(e, percent_mismatch):
continue
logger.error(e)
@ -2051,7 +2051,7 @@ class ParseDictArgAction(argparse.Action):
for kv in values.split(","):
try:
k, v = kv.split("=")
except ValueError: # noqa: PERF203
except ValueError:
parser.error(f"argument {option_string}: Expected '=' between key and value")
if k in dict_arg:

View file

@ -146,7 +146,7 @@ def get_memory(memory, model_group):
memory_columns = [model_title]
for provider in provider_list:
if cpu not in provider:
memory_columns.append(provider + memory_ending) # noqa: PERF401
memory_columns.append(provider + memory_ending)
memory_db_columns = [
model_title,
cuda,
@ -273,7 +273,7 @@ def get_latency(latency, model_group):
latency_columns = [model_title]
for provider in provider_list:
latency_columns.append(provider + avg_ending) # noqa: PERF401
latency_columns.append(provider + avg_ending)
latency_db_columns = table_headers
latency = adjust_columns(latency, latency_columns, latency_db_columns, model_group)
return latency

View file

@ -75,7 +75,7 @@ def main():
model_list = []
for link in links:
model_list.append(get_model_info(link)) # noqa: PERF401
model_list.append(get_model_info(link))
write_json(model_list)

View file

@ -417,7 +417,7 @@ def run_pytorch(
result.update(get_latency_result(runtimes, batch_size))
logger.info(result)
results.append(result)
except RuntimeError as e: # noqa: PERF203
except RuntimeError as e:
logger.exception(e)
torch.cuda.empty_cache()
@ -572,7 +572,7 @@ def run_tensorflow(
result.update(get_latency_result(runtimes, batch_size))
logger.info(result)
results.append(result)
except RuntimeError as e: # noqa: PERF203
except RuntimeError as e:
logger.exception(e)
from numba import cuda

View file

@ -249,7 +249,7 @@ def output_summary(results, csv_filename, args):
data_names.append(f"b{batch_size}")
else:
for sequence_length in args.sequence_lengths:
data_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401
data_names.append(f"b{batch_size}_s{sequence_length}")
csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + data_names)
csv_writer.writeheader()
@ -386,7 +386,7 @@ def allocateOutputBuffers(output_buffers, output_buffer_max_sizes, device): # n
# for each test run.
for i in output_buffer_max_sizes:
output_buffers.append(torch.empty(i, dtype=torch.float32, device=device)) # noqa: PERF401
output_buffers.append(torch.empty(i, dtype=torch.float32, device=device))
def set_random_seed(seed=123):

View file

@ -197,7 +197,7 @@ def tf2pt_pipeline_test():
input = torch.randint(low=0, high=config.vocab_size - 1, size=(4, 128), dtype=torch.long)
try:
model(input)
except RuntimeError as e: # noqa: PERF203
except RuntimeError as e:
logger.exception(e)

View file

@ -124,7 +124,7 @@ class PackingMode:
attributes = []
for attr in attention.attribute:
if attr.name in ["num_heads", "qkv_hidden_sizes", "scale"]:
attributes.append(attr) # noqa: PERF401
attributes.append(attr)
packed_attention.attribute.extend(attributes)
packed_attention.domain = "com.microsoft"

View file

@ -150,7 +150,7 @@ def output_summary(results: List[Dict[str, Any]], csv_filename: str, metric_name
key_names = []
for sequence_length in sequence_lengths:
for batch_size in batch_sizes:
key_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401
key_names.append(f"b{batch_size}_s{sequence_length}")
csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + key_names)
csv_writer.writeheader()

View file

@ -364,7 +364,7 @@ def main(args):
# Results of IO binding might be in GPU. Copy outputs to CPU for comparison.
copy_outputs = []
for output in ort_outputs:
copy_outputs.append(output.cpu().numpy()) # noqa: PERF401
copy_outputs.append(output.cpu().numpy())
if gpt2helper.compare_outputs(
outputs,
@ -404,7 +404,7 @@ def main(args):
"onnxruntime_latency": f"{ort_latency:.2f}",
}
csv_writer.writerow(row)
except Exception: # noqa: PERF203
except Exception:
logger.error("Exception", exc_info=True)
return None

View file

@ -75,7 +75,7 @@ class MyGPT2Model(GPT2Model):
for i in range(num_layer):
# Since transformers v4.*, past key and values are separated outputs.
# Here we concate them into one tensor to be compatible with Attention operator.
present.append( # noqa: PERF401
present.append(
torch.cat(
(result[1][i][0].unsqueeze(0), result[1][i][1].unsqueeze(0)),
dim=0,

View file

@ -256,7 +256,7 @@ def run_significance_test(rows, output_csv_path):
utest_statistic, utest_pvalue = scipy.stats.mannwhitneyu(
a, b, use_continuity=True, alternative="two-sided"
) # TODO: shall we use one-sided: less or greater according to "top1_match_rate"
except ValueError: # ValueError: All numbers are identical in mannwhitneyu # noqa: PERF203
except ValueError: # ValueError: All numbers are identical in mannwhitneyu
utest_statistic = None
utest_pvalue = None
ttest_statistic, ttest_pvalue = scipy.stats.ttest_ind(a, b, axis=None, equal_var=True)

View file

@ -645,7 +645,7 @@ def run_tests(
args = parse_arguments(f"{arguments} -t {test_times}".split(" "))
latency_results = launch_test(args)
except KeyboardInterrupt as exc: # noqa: PERF203
except KeyboardInterrupt as exc:
raise RuntimeError("Keyboard Interrupted") from exc
except Exception:
traceback.print_exc()
@ -687,7 +687,7 @@ def output_summary(results, csv_filename, data_field="average_latency_ms"):
data_names = []
for sequence_length in sequence_lengths:
for batch_size in batch_sizes:
data_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401
data_names.append(f"b{batch_size}_s{sequence_length}")
csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + data_names)
csv_writer.writeheader()

View file

@ -204,10 +204,10 @@ class T5DecoderInputs:
past = []
for _ in range(2 * num_layers):
past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401
past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device))
for _ in range(2 * num_layers):
past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401
past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device))
else:
past = None

View file

@ -167,10 +167,10 @@ class WhisperDecoderInputs:
past = []
for _ in range(2 * num_layers):
past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401
past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device))
for _ in range(2 * num_layers):
past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401
past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device))
else:
past = None

View file

@ -108,14 +108,14 @@ class OnnxModel:
input_names = []
for graph in self.graphs():
for input in graph.input:
input_names.append(input.name) # noqa: PERF401
input_names.append(input.name)
return input_names
def get_graphs_output_names(self):
output_names = []
for graph in self.graphs():
for output in graph.output:
output_names.append(output.name) # noqa: PERF401
output_names.append(output.name)
return output_names
def get_graph_by_node(self, node):
@ -217,7 +217,7 @@ class OnnxModel:
nodes = []
for node in self.nodes():
if node.op_type == op_type:
nodes.append(node) # noqa: PERF401
nodes.append(node)
return nodes
def get_children(self, node, input_name_to_nodes=None):
@ -238,7 +238,7 @@ class OnnxModel:
parents = []
for input in node.input:
if input in output_name_to_node:
parents.append(output_name_to_node[input]) # noqa: PERF401
parents.append(output_name_to_node[input])
return parents
def get_parent(self, node, i, output_name_to_node=None):
@ -792,7 +792,7 @@ class OnnxModel:
nodes = self.nodes()
for node in nodes:
if node.op_type == "Constant" and node.output[0] not in input_name_to_nodes:
unused_nodes.append(node) # noqa: PERF401
unused_nodes.append(node)
self.remove_nodes(unused_nodes)
@ -837,7 +837,7 @@ class OnnxModel:
output_to_remove = []
for output in self.model.graph.output:
if output.name not in outputs:
output_to_remove.append(output) # noqa: PERF401
output_to_remove.append(output)
for output in output_to_remove:
self.model.graph.output.remove(output)
@ -882,7 +882,7 @@ class OnnxModel:
if allow_remove_graph_inputs:
for input in graph.input:
if input.name not in remaining_input_names:
inputs_to_remove.append(input) # noqa: PERF401
inputs_to_remove.append(input)
for input in inputs_to_remove:
graph.input.remove(input)
@ -1058,7 +1058,7 @@ class OnnxModel:
graph_inputs = []
for input in self.model.graph.input:
if self.get_initializer(input.name) is None:
graph_inputs.append(input) # noqa: PERF401
graph_inputs.append(input)
return graph_inputs
def get_opset_version(self):

View file

@ -47,7 +47,7 @@ class UnetOnnxModel(BertOnnxModel):
nodes_to_remove = []
for div in div_nodes:
if self.find_constant_input(div, 1.0) == 1:
nodes_to_remove.append(div) # noqa: PERF401
nodes_to_remove.append(div)
for node in nodes_to_remove:
self.replace_input_of_all_nodes(node.output[0], node.input[0])

View file

@ -78,7 +78,7 @@ class BertOnnxModelShapeOptimizer(OnnxModel):
shape_inputs = []
for node in self.model.graph.node:
if node.op_type == "Reshape":
shape_inputs.append(node.input[1]) # noqa: PERF401
shape_inputs.append(node.input[1])
return shape_inputs

View file

@ -406,7 +406,7 @@ class TestInferenceSession(unittest.TestCase):
run_base_test2()
run_advanced_test()
except OSError: # noqa: PERF203
except OSError:
continue
else:
break

View file

@ -35,7 +35,7 @@ class TestDataReader(CalibrationDataReader):
self.count = 4
self.input_data_list = []
for _ in range(self.count):
self.input_data_list.append(np.random.normal(0, 0.33, [1, 3, 1, 3]).astype(np.float32)) # noqa: PERF401
self.input_data_list.append(np.random.normal(0, 0.33, [1, 3, 1, 3]).astype(np.float32))
def get_next(self):
if self.preprocess_flag:

View file

@ -93,7 +93,7 @@ class TestDataReader(CalibrationDataReader):
self.count = 2
self.input_data_list = []
for _ in range(self.count):
self.input_data_list.append(np.random.normal(0, 0.33, input_shape).astype(np.float32)) # noqa: PERF401
self.input_data_list.append(np.random.normal(0, 0.33, input_shape).astype(np.float32))
def get_next(self):
if self.preprocess_flag:
@ -144,7 +144,7 @@ class TestSaveActivations(unittest.TestCase):
data_reader.rewind()
oracle_outputs = []
for input_d in data_reader:
oracle_outputs.append(infer_session.run(None, input_d)) # noqa: PERF401
oracle_outputs.append(infer_session.run(None, input_d))
output_dict = {}
output_info = infer_session.get_outputs()

View file

@ -342,7 +342,7 @@ def generate_test_data(
path = os.path.join(output_path, "test_data_set_" + str(test_case))
try:
os.mkdir(path)
except OSError: # noqa: PERF203
except OSError:
print("Creation of the directory %s failed" % path)
else:
print("Successfully created the directory %s " % path)

View file

@ -451,7 +451,7 @@ def generate_test_data(
path = os.path.join(output_path, "test_data_set_" + str(test_case))
try:
os.mkdir(path)
except OSError: # noqa: PERF203
except OSError:
print("Creation of the directory %s failed" % path)
else:
print("Successfully created the directory %s " % path)

View file

@ -145,7 +145,7 @@ def _order_paths(paths, D_groups, H_groups):
world_rank = _utils.state_dict_trainer_options_world_rank_key()
for path in paths:
trainer_options_path_tuples.append( # noqa: PERF401
trainer_options_path_tuples.append(
(_checkpoint_storage.load(path, key=_utils.state_dict_trainer_options_key()), path)
)
@ -365,7 +365,7 @@ def _get_parallellism_groups(data_parallel_size, horizontal_parallel_size, world
for data_group_id in range(num_data_groups):
data_group_ranks = []
for r in range(data_parallel_size):
data_group_ranks.append(data_group_id + horizontal_parallel_size * r) # noqa: PERF401
data_group_ranks.append(data_group_id + horizontal_parallel_size * r)
data_groups.append(data_group_ranks)
num_horizontal_groups = world_size // horizontal_parallel_size
@ -373,7 +373,7 @@ def _get_parallellism_groups(data_parallel_size, horizontal_parallel_size, world
for hori_group_id in range(num_horizontal_groups):
hori_group_ranks = []
for r in range(horizontal_parallel_size):
hori_group_ranks.append(hori_group_id * horizontal_parallel_size + r) # noqa: PERF401
hori_group_ranks.append(hori_group_id * horizontal_parallel_size + r)
horizontal_groups.append(hori_group_ranks)
return data_groups, horizontal_groups

View file

@ -187,7 +187,7 @@ class AdamW(onnxblock_module.ForwardBlock):
# Prepare the tensor sequence inputs for params and moments
for input_name in [params_name, gradients_name, first_order_moments_name, second_order_moments_name]:
onnx_model.graph.input.append( # noqa: PERF401
onnx_model.graph.input.append(
onnx.helper.make_tensor_sequence_value_info(input_name, trainable_parameters[0].data_type, None)
)

View file

@ -294,7 +294,7 @@ class GraphLowering:
producers[output] = node
for input in node.input:
if input in producers:
precessors[node.name].append(producers[input]) # noqa: PERF401
precessors[node.name].append(producers[input])
for value in precessors.values():
value.sort(key=sorted_nodes.index, reverse=True)
for idx in range(len(sorted_nodes) - 1, -1, -1):
@ -441,9 +441,7 @@ class GraphLowering:
assert isinstance(sub_nodes[nxt], ReduceForLoopEnd)
for reduce_node in sub_nodes[nxt].reduce_nodes:
if reduce_node.outputs[0].name in output_name_map:
reduce_store_nodes.append( # noqa: PERF401
IONode(reduce_node.outputs[0], kernel_node.offset_calc, False)
)
reduce_store_nodes.append(IONode(reduce_node.outputs[0], kernel_node.offset_calc, False))
new_sub_nodes.append(sub_nodes[nxt])
nxt += 1
cur = nxt

View file

@ -110,7 +110,7 @@ class SortedGraph:
for node_idx, node in enumerate(self._sorted_nodes):
inputs = []
for input in node.input:
inputs.append(name_map.get(input, input)) # noqa: PERF401
inputs.append(name_map.get(input, input))
inputs_str = ",".join(inputs)
outputs = []
for idx, output in enumerate(node.output):
@ -180,7 +180,7 @@ class SortedGraph:
else:
input_infos = []
for input in node.input:
input_infos.append(self._node_arg_infos[input]) # noqa: PERF401
input_infos.append(self._node_arg_infos[input])
output_infos = TypeAndShapeInfer.infer(node, input_infos, self._graph)
for idx, output in enumerate(node.output):
self._node_arg_infos[output] = output_infos[idx]

View file

@ -52,7 +52,7 @@ def topological_sort(inputs: List[str], nodes: List[NodeProto]) -> List[NodeProt
continue
for consumer in non_const_nodes:
if output in consumer.input:
output_consumers[node.name].append(consumer) # noqa: PERF401
output_consumers[node.name].append(consumer)
# Topological sort.
visited = set()

View file

@ -357,7 +357,7 @@ def transform_slice_scel(graph):
all_nodes = []
for node in graph.node:
if node not in remove_nodes:
all_nodes.append(node) # noqa: PERF401
all_nodes.append(node)
for node in triton_nodes:
all_nodes.append(node) # noqa: PERF402

View file

@ -36,7 +36,7 @@ class _IteratedORTModule(torch.nn.Module):
self._it = count - 1
self._ortmodules = []
for idx in range(count):
self._ortmodules.append( # noqa: PERF401
self._ortmodules.append(
ORTModule(
module,
debug_options=DebugOptions(

View file

@ -20,7 +20,7 @@ def _list_extensions(path):
for root, _, files in os.walk(path):
for name in files:
if name.lower() == "setup.py":
extensions.append(os.path.join(root, name)) # noqa: PERF401
extensions.append(os.path.join(root, name))
return extensions

View file

@ -933,7 +933,7 @@ class ORTTrainer:
# so output will be on the same device as input.
try:
torch.device(target_device)
except Exception: # noqa: PERF203
except Exception:
# in this case, input/output must on CPU
assert input.device.type == "cpu"
target_device = "cpu"

View file

@ -26,7 +26,7 @@ def find_input_node(model, arg):
for node in model.graph.node:
for output in node.output:
if output == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -35,7 +35,7 @@ def find_output_node(model, arg):
for node in model.graph.node:
for input in node.input:
if input == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else result
@ -189,7 +189,7 @@ def find_nodes(graph, op_type):
nodes = []
for node in graph.node:
if node.op_type == op_type:
nodes.append(node) # noqa: PERF401
nodes.append(node)
return nodes
@ -382,7 +382,7 @@ def layer_norm_transform(model):
all_nodes = []
for node in graph.node:
if node not in remove_nodes:
all_nodes.append(node) # noqa: PERF401
all_nodes.append(node)
for node in layer_norm_nodes:
all_nodes.append(node) # noqa: PERF402

View file

@ -64,7 +64,7 @@ class Test_PostPasses(unittest.TestCase): # noqa: N801
nodes = []
for node in model.graph.node:
if node.op_type == node_type:
nodes.append(node) # noqa: PERF401
nodes.append(node)
return nodes
def get_name(self, name):

View file

@ -20,7 +20,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None):
values = []
for _ in range(total_dims):
values.append(rng.randint(0, vocab_size - 1)) # noqa: PERF401
values.append(rng.randint(0, vocab_size - 1))
return torch.tensor(data=values, dtype=torch.long).view(shape).contiguous()
@ -36,7 +36,7 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None):
values = []
for _ in range(total_dims):
values.append(rng.random() * scale) # noqa: PERF401
values.append(rng.random() * scale)
return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous()

View file

@ -213,7 +213,7 @@ def test_hierarchical_ortmodule():
call_backward(y_ref)
g_ref = []
for param in m.parameters():
g_ref.append(param.grad.detach()) # noqa: PERF401
g_ref.append(param.grad.detach())
m.zero_grad()
@ -224,7 +224,7 @@ def test_hierarchical_ortmodule():
call_backward(y)
g = []
for param in m.parameters():
g.append(param.grad.detach()) # noqa: PERF401
g.append(param.grad.detach())
# Some sub-modules become ORTModule.
assert expected_num_ortmodule == count_ortmodule(m)

View file

@ -173,7 +173,7 @@ def layer_norm_transform(model_proto):
all_nodes = []
for node in graph_proto.node:
if node not in removed_nodes:
all_nodes.append(node) # noqa: PERF401
all_nodes.append(node)
for node in layer_norm_nodes:
all_nodes.append(node) # noqa: PERF402

View file

@ -13,7 +13,7 @@ def find_single_output_node(model, arg):
for node in model.graph.node:
for input in node.input:
if input == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -63,7 +63,7 @@ def fix_transpose(model):
for n in model.graph.node:
for input in n.input:
if input == weight.name:
result.append(n) # noqa: PERF401
result.append(n)
if len(result) > 1:
continue
perm = node.attribute[0]
@ -93,7 +93,7 @@ def fix_transpose(model):
old_ws = []
for t in transpose:
if find_single_output_node(model, t[1].name) is None:
old_ws.append(find_weight_index(model, t[1].name)) # noqa: PERF401
old_ws.append(find_weight_index(model, t[1].name))
old_ws.sort(reverse=True)
for w_i in old_ws:
del model.graph.initializer[w_i]

View file

@ -1497,10 +1497,10 @@ def test_gradient_correctness_einsum(equation):
rhs_op = equation[pos1 + 1 : pos2]
lhs_shape = []
for c in lhs_op:
lhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401
lhs_shape.append(SIZE_MAP[c.upper()])
rhs_shape = []
for c in rhs_op:
rhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401
rhs_shape.append(SIZE_MAP[c.upper()])
pt_model = NeuralNetEinsum(lhs_shape[-1]).to(device)
ort_model = ORTModule(copy.deepcopy(pt_model))
@ -1577,7 +1577,7 @@ def test_gradient_correctness_einsum_2():
random.shuffle(output_candidates)
output_candidates = output_candidates[:8]
for output_candidate in [list(candidate) for candidate in output_candidates]:
all_cases.append((lhs_candidate, rhs_candidate, output_candidate)) # noqa: PERF401
all_cases.append((lhs_candidate, rhs_candidate, output_candidate))
for case in all_cases:
equation = to_string(case[0]) + "," + to_string(case[1]) + "->" + to_string(case[2])
@ -1587,10 +1587,10 @@ def test_gradient_correctness_einsum_2():
rhs_op = equation[pos1 + 1 : pos2]
lhs_shape = []
for c in lhs_op:
lhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401
lhs_shape.append(SIZE_MAP[c.upper()])
rhs_shape = []
for c in rhs_op:
rhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401
rhs_shape.append(SIZE_MAP[c.upper()])
pt_model = NeuralNetEinsum(lhs_shape[-1]).to(device)
ort_model = ORTModule(copy.deepcopy(pt_model))
@ -5895,7 +5895,7 @@ def test_ops_for_padding_elimination(test_cases):
result = []
for node in model.graph.node:
if arg in node.output:
result.append(node) # noqa: PERF401
result.append(node)
return result[0].op_type if len(result) == 1 else None
gathergrad_input_optypes = [find_input_node_type(training_model, arg) for arg in gathergrad_node.input]

View file

@ -112,7 +112,7 @@ def optimizer_parameters(model):
no_decay_param_group = []
for initializer in model.graph.initializer:
if any(key in initializer.name for key in no_decay_keys):
no_decay_param_group.append(initializer.name) # noqa: PERF401
no_decay_param_group.append(initializer.name)
params = [
{
"params": no_decay_param_group,

View file

@ -15,7 +15,7 @@ def get_gpu_lines(path):
reader = csv.reader(f, delimiter=",")
for row in reader:
if row[2].find("TotalDurationNs") < 0:
lines.append(row) # noqa: PERF401
lines.append(row)
return lines

View file

@ -28,7 +28,7 @@ def find_input_node(model, arg):
for node in model.graph.node:
for output in node.output:
if output == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -37,7 +37,7 @@ def find_output_node(model, arg):
for node in model.graph.node:
for input in node.input:
if input == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -136,7 +136,7 @@ def process_concat(model):
assert reshape_node.op_type == "Reshape"
new_nodes[get_node_index(model, reshape_node)] = shape
for n in fuse_nodes:
delete_nodes.append(get_node_index(model, n)) # noqa: PERF401
delete_nodes.append(get_node_index(model, n))
# insert new shape to reshape
index = 0
@ -189,7 +189,7 @@ def fix_transpose(model):
for n in model.graph.node:
for input in n.input:
if input == weight.name:
result.append(n) # noqa: PERF401
result.append(n)
if len(result) > 1:
continue
perm = node.attribute[0]
@ -280,7 +280,7 @@ def remove_input_ids_check_subgraph(model):
remove_node_index = []
for n in removed_nodes:
remove_node_index.append(get_node_index(model, n)) # noqa: PERF401
remove_node_index.append(get_node_index(model, n))
remove_node_index = list(set(remove_node_index))
remove_node_index.sort(reverse=True)

View file

@ -141,7 +141,7 @@ def main():
all_nodes = []
for node in graph_proto.node:
if node not in removed_nodes:
all_nodes.append(node) # noqa: PERF401
all_nodes.append(node)
for node in layer_norm_nodes:
all_nodes.append(node) # noqa: PERF402

View file

@ -26,7 +26,7 @@ def find_input_node(model, arg):
for node in model.graph.node:
for output in node.output:
if output == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -35,7 +35,7 @@ def find_output_node(model, arg):
for node in model.graph.node:
for input in node.input:
if input == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None
@ -94,7 +94,7 @@ def process_concat(model):
if node.op_type == "Concat":
input_nodes = []
for input in node.input:
input_nodes.append(find_input_node(model, input)) # noqa: PERF401
input_nodes.append(find_input_node(model, input))
# figure out target shape
shape = []
for input_node in input_nodes:
@ -116,7 +116,7 @@ def process_concat(model):
assert reshape_node.op_type == "Reshape"
new_nodes[get_node_index(model, reshape_node)] = shape
for n in fuse_nodes:
delete_nodes.append(get_node_index(model, n)) # noqa: PERF401
delete_nodes.append(get_node_index(model, n))
# insert new shape to reshape
index = 0
for reshape_node_index in new_nodes:
@ -218,7 +218,7 @@ def fix_transpose(model):
for n in model.graph.node:
for input in n.input:
if input == weight.name:
result.append(n) # noqa: PERF401
result.append(n)
if len(result) > 1:
continue
perm = node.attribute[0]
@ -242,7 +242,7 @@ def fix_transpose(model):
old_ws = []
for t in transpose:
if find_output_node(model, t[1].name) is None:
old_ws.append(find_weight_index(model, t[1].name)) # noqa: PERF401
old_ws.append(find_weight_index(model, t[1].name))
old_ws.sort(reverse=True)
for w_i in old_ws:
del model.graph.initializer[w_i]

View file

@ -34,7 +34,7 @@ def find_input_node(model, arg):
for node in model.graph.node:
for output in node.output:
if output == arg:
result.append(node) # noqa: PERF401
result.append(node)
return result[0] if len(result) == 1 else None

View file

@ -30,11 +30,11 @@ def process_file(onnx_file):
if node.op_type == "ATen":
for attr in node.attribute:
if attr.name == "operator":
aten_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") # noqa: PERF401
aten_ops.append(f"{node.name}: {attr.s.decode('utf-8')}")
if node.op_type == "PythonOp":
for attr in node.attribute:
if attr.name == "name":
python_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") # noqa: PERF401
python_ops.append(f"{node.name}: {attr.s.decode('utf-8')}")
# Look for stand-alone Dropout node in *_execution_model_<mode>.onnx graph.
# Examine whether it should be fused with surrounding Add ops into BiasDropout node.

View file

@ -49,7 +49,7 @@ def split_graph(model, split_edge_groups):
element_types.append(1)
for info in model.graph.value_info:
if info.name == id:
output_shapes.append(info.type) # noqa: PERF401
output_shapes.append(info.type)
send_input_signal_name = "send_input_signal" + str(cut_index)
send_signal = model.graph.input.add()
@ -279,14 +279,14 @@ def generate_subgraph(model, start_nodes, identity_node_list):
# remove added identity node before copy to subgraph
identity_node_index = []
for n in identity_node_list:
identity_node_index.append(get_identity_index_for_deleting(main_graph.graph.node, n)) # noqa: PERF401
identity_node_index.append(get_identity_index_for_deleting(main_graph.graph.node, n))
identity_node_index.sort(reverse=True)
for i in reversed(range(len(main_graph.graph.node))):
try:
if i in identity_node_index:
del main_graph.graph.node[i]
except Exception: # noqa: PERF203
except Exception:
print("error deleting identity node", i)
all_visited_nodes = []
@ -316,19 +316,19 @@ def generate_subgraph(model, start_nodes, identity_node_list):
# gather visited nodes
visited_nodes = []
for n in visited0:
visited_nodes.append(get_index(main_graph.graph.node, n)) # noqa: PERF401
visited_nodes.append(get_index(main_graph.graph.node, n))
visited_nodes.sort(reverse=True)
# gather visited inputs
visited_inputs = []
for n in inputs0:
visited_inputs.append(get_index(main_graph.graph.input, n)) # noqa: PERF401
visited_inputs.append(get_index(main_graph.graph.input, n))
visited_inputs.sort(reverse=True)
# gather visited outputs
visited_outputs = []
for n in outputs0:
visited_outputs.append(get_index(main_graph.graph.output, n)) # noqa: PERF401
visited_outputs.append(get_index(main_graph.graph.output, n))
visited_outputs.sort(reverse=True)
for i in reversed(range(len(main_graph.graph.node))):
@ -337,7 +337,7 @@ def generate_subgraph(model, start_nodes, identity_node_list):
del subgraph.graph.node[i]
else:
del main_graph.graph.node[i]
except Exception: # noqa: PERF203
except Exception:
print("error deleting node", i)
for i in reversed(range(len(main_graph.graph.input))):
@ -346,7 +346,7 @@ def generate_subgraph(model, start_nodes, identity_node_list):
del subgraph.graph.input[i]
else:
del main_graph.graph.input[i]
except Exception: # noqa: PERF203
except Exception:
print("error deleting inputs", i)
for i in reversed(range(len(main_graph.graph.output))):
@ -355,7 +355,7 @@ def generate_subgraph(model, start_nodes, identity_node_list):
del subgraph.graph.output[i]
else:
del main_graph.graph.output[i]
except Exception: # noqa: PERF203
except Exception:
print("error deleting outputs ", i)
print("model", str(model_count), " length ", len(subgraph.graph.node))

View file

@ -71,6 +71,8 @@ ignore = [
"N812", # Allow import torch.nn.functional as F
"N999", # Module names
"NPY002", # np.random.Generator may not always fit our use cases
"PERF203", # "try-except-in-loop" only affects Python <3.11, and the improvement is minor; can have false positives
"PERF401", # List comprehensions are not always readable
"SIM102", # We don't perfer always combining if branches
"SIM108", # We don't encourage ternary operators
"SIM114", # Don't combine if branches for debugability

View file

@ -69,7 +69,7 @@ def write_to_db(binary_size_data, args):
branch_name = os.environ.get("BUILD_SOURCEBRANCHNAME", "main")
rows = []
for row in binary_size_data:
rows.append( # noqa: PERF401
rows.append(
[
now_str,
args.build_id,

View file

@ -16,7 +16,7 @@ def rename_folder(root):
for r, dirs, _files in os.walk(root):
for name in dirs:
if name.startswith("_"):
found.append((r, name)) # noqa: PERF401
found.append((r, name))
renamed = []
for r, name in found:
into = name.lstrip("_")

View file

@ -87,9 +87,7 @@ def generate_file_list_for_ep(nuget_artifacts_dir, ep, files_list, include_pdbs,
if child.name == "onnxruntime-android" or child.name == "onnxruntime-training-android":
for child_file in child.iterdir():
if child_file.suffix in [".aar"]:
files_list.append( # noqa: PERF401
'<file src="' + str(child_file) + '" target="runtimes/android/native"/>'
)
files_list.append('<file src="' + str(child_file) + '" target="runtimes/android/native"/>')
if child.name == "onnxruntime-ios-xcframework":
files_list.append('<file src="' + str(child) + "\\**" '" target="runtimes/ios/native"/>') # noqa: ISC001
@ -722,7 +720,7 @@ def generate_files(line_list, args):
ngraph_list_path = os.path.join(openvino_path, "deployment_tools\\ngraph\\lib\\")
for ngraph_element in os.listdir(ngraph_list_path):
if ngraph_element.endswith("dll"):
files_list.append( # noqa: PERF401
files_list.append(
"<file src="
+ '"'
+ os.path.join(ngraph_list_path, ngraph_element)
@ -732,7 +730,7 @@ def generate_files(line_list, args):
)
for dll_element in os.listdir(dll_list_path):
if dll_element.endswith("dll"):
files_list.append( # noqa: PERF401
files_list.append(
"<file src="
+ '"'
+ os.path.join(dll_list_path, dll_element)
@ -762,7 +760,7 @@ def generate_files(line_list, args):
)
for tbb_element in os.listdir(tbb_list_path):
if tbb_element.endswith("dll"):
files_list.append( # noqa: PERF401
files_list.append(
"<file src="
+ '"'
+ os.path.join(tbb_list_path, tbb_element)

View file

@ -33,7 +33,7 @@ class OrtFormatModelDumper:
tensor = graph.Initializers(idx)
dims = []
for dim in range(0, tensor.DimsLength()):
dims.append(tensor.Dims(dim)) # noqa: PERF401
dims.append(tensor.Dims(dim))
print(f"{tensor.Name().decode()} data_type={tensor.DataType()} dims={dims}")
print("--------")

View file

@ -117,7 +117,7 @@ def parse(graph: GraphProto) -> GraphDef:
for node in graph.node:
_attr = []
for s in node.attribute:
_attr.append(" = ".join([str(f[1]) for f in s.ListFields()])) # noqa: PERF401
_attr.append(" = ".join([str(f[1]) for f in s.ListFields()]))
attr = ", ".join(_attr).encode(encoding="utf_8")
shape_proto = None
elem_type = 0
@ -331,7 +331,7 @@ class ListUnpackTransformer(TransformerBase):
new_output = f"{get_prefix(node.output[0])}{node.op_type}_{idx!s}_output"
for output in node.output:
if len(output) > 0:
new_nodes.append(helper.make_node("ListUnpack", [new_output], [output])) # noqa: PERF401
new_nodes.append(helper.make_node("ListUnpack", [new_output], [output]))
node.ClearField("output")
node.output.extend([new_output])
if len(new_nodes) > 0:

View file

@ -165,7 +165,7 @@ def _convert(
# new_size = os.path.getsize(ort_target_path)
# print("Serialized {} to {}. Sizes: orig={} new={} diff={} new:old={:.4f}:1.0".format(
# onnx_target_path, ort_target_path, orig_size, new_size, new_size - orig_size, new_size / orig_size))
except Exception as e: # noqa: PERF203
except Exception as e:
print(f"Error converting {model}: {e}")
if not allow_conversion_failures:
raise

View file

@ -205,7 +205,7 @@ class DefaultTypeUsageProcessor(TypeUsageProcessor):
domain = _ort_constant_for_domain(self.domain)
for i in sorted(self._input_types.keys()):
if self._input_types[i]:
entries.append( # noqa: PERF401
entries.append(
"ORT_SPECIFY_OP_KERNEL_ARG_ALLOWED_TYPES({}, {}, Input, {}, {});".format(
domain, self.optype, i, ", ".join(sorted(self._input_types[i]))
)
@ -213,7 +213,7 @@ class DefaultTypeUsageProcessor(TypeUsageProcessor):
for o in sorted(self._output_types.keys()):
if self._output_types[o]:
entries.append( # noqa: PERF401
entries.append(
"ORT_SPECIFY_OP_KERNEL_ARG_ALLOWED_TYPES({}, {}, Output, {}, {});".format(
domain, self.optype, o, ", ".join(sorted(self._output_types[o]))
)