From 0c1a5098dc3417c7bf244e7b19f716164bd3aacc Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 25 Jul 2023 15:38:22 -0700 Subject: [PATCH] Disable PERF* rules in ruff to allow better readability (#16834) ### Description Disable two PERF* rules in ruff to allow better readability. Rational commented inline. This change also removes the unused noqa directives because of the rule change. ### Motivation and Context Readability --- docs/python/_common/onnx_sphinx.py | 2 +- docs/python/examples/plot_common_errors.py | 6 +++--- onnxruntime/python/backend/backend.py | 2 +- .../python/onnxruntime_inference_collection.py | 2 +- .../create_custom_op_wrapper.py | 2 +- .../kernels/batched_gemm_test.py | 2 +- .../kernels/gemm_softmax_gemm_permute_test.py | 2 +- .../tools/kernel_explorer/kernels/gemm_test.py | 2 +- .../kernels/strided_batched_gemm_test.py | 2 +- .../python/tools/pytorch_export_contrib_ops.py | 2 +- .../python/tools/quantization/calibrate.py | 2 +- .../python/tools/quantization/onnx_model.py | 6 +++--- .../tools/quantization/qdq_loss_debug.py | 2 +- .../python/tools/symbolic_shape_infer.py | 6 +++--- .../python/tools/tensorrt/perf/benchmark.py | 10 +++++----- onnxruntime/python/tools/tensorrt/perf/post.py | 4 ++-- .../perf/setup_scripts/setup_onnx_zoo.py | 2 +- .../python/tools/transformers/benchmark.py | 4 ++-- .../tools/transformers/benchmark_helper.py | 4 ++-- .../convert_tf_models_to_pytorch.py | 2 +- .../transformers/convert_to_packing_mode.py | 2 +- .../transformers/models/bert/eval_squad.py | 2 +- .../transformers/models/gpt2/benchmark_gpt2.py | 4 ++-- .../transformers/models/gpt2/gpt2_helper.py | 2 +- .../transformers/models/gpt2/gpt2_parity.py | 2 +- .../models/longformer/benchmark_longformer.py | 4 ++-- .../tools/transformers/models/t5/t5_decoder.py | 4 ++-- .../models/whisper/whisper_decoder.py | 4 ++-- .../python/tools/transformers/onnx_model.py | 16 ++++++++-------- .../tools/transformers/onnx_model_unet.py | 2 +- .../tools/transformers/shape_optimizer.py | 2 +- .../test/python/onnxruntime_test_python.py | 2 +- .../python/quantization/test_calibration.py | 2 +- .../python/quantization/test_qdq_loss_debug.py | 4 ++-- .../generate_tiny_keras2onnx_bert_models.py | 2 +- .../generate_tiny_gpt2_model.py | 2 +- .../orttraining/python/training/checkpoint.py | 6 +++--- .../python/training/onnxblock/optim/optim.py | 2 +- .../python/training/ort_triton/_lowering.py | 6 ++---- .../training/ort_triton/_sorted_graph.py | 4 ++-- .../python/training/ort_triton/_utils.py | 2 +- .../training/ort_triton/kernel/_slice_scel.py | 2 +- .../_hierarchical_ortmodule.py | 2 +- .../ortmodule/torch_cpp_extensions/install.py | 2 +- .../orttraining/python/training/orttrainer.py | 2 +- .../orttraining/python/training/postprocess.py | 8 ++++---- .../python/onnxruntime_test_postprocess.py | 2 +- .../python/orttraining_test_data_loader.py | 4 ++-- .../orttraining_test_hierarchical_ortmodule.py | 4 ++-- .../orttraining_test_layer_norm_transform.py | 2 +- .../python/orttraining_test_model_transform.py | 6 +++--- .../python/orttraining_test_ortmodule_api.py | 12 ++++++------ ...rttraining_test_orttrainer_bert_toy_onnx.py | 2 +- orttraining/tools/amdgpu/script/rocprof.py | 2 +- .../tools/scripts/gpt2_model_transform.py | 10 +++++----- .../tools/scripts/layer_norm_transform.py | 2 +- orttraining/tools/scripts/model_transform.py | 12 ++++++------ .../tools/scripts/opset12_model_transform.py | 2 +- .../tools/scripts/performance_investigation.py | 4 ++-- .../tools/scripts/pipeline_model_split.py | 18 +++++++++--------- pyproject.toml | 2 ++ .../windows/post_binary_sizes_to_dashboard.py | 2 +- tools/doc/rename_folders.py | 2 +- .../nuget/generate_nuspec_for_native_nuget.py | 10 ++++------ tools/python/dump_ort_model.py | 2 +- tools/python/onnx2tfevents.py | 4 ++-- .../python/util/convert_onnx_models_to_ort.py | 2 +- .../operator_type_usage_processors.py | 4 ++-- 68 files changed, 133 insertions(+), 135 deletions(-) diff --git a/docs/python/_common/onnx_sphinx.py b/docs/python/_common/onnx_sphinx.py index dcebf2ced0..7562d23289 100644 --- a/docs/python/_common/onnx_sphinx.py +++ b/docs/python/_common/onnx_sphinx.py @@ -683,7 +683,7 @@ def get_onnx_example(op_name): try: mod = importlib.import_module(m) module = m - except ImportError: # noqa: PERF203 + except ImportError: continue if module is None: # Unable to find an example for 'op_name'. diff --git a/docs/python/examples/plot_common_errors.py b/docs/python/examples/plot_common_errors.py index a121f8ba6c..dc7078831a 100644 --- a/docs/python/examples/plot_common_errors.py +++ b/docs/python/examples/plot_common_errors.py @@ -86,7 +86,7 @@ for x in [ try: r = sess.run([output_name], {input_name: x}) print(f"Shape={x.shape} and predicted labels={r}") - except (RuntimeError, InvalidArgument) as e: # noqa: PERF203 + except (RuntimeError, InvalidArgument) as e: print(f"ERROR with Shape={x.shape} - {e}") for x in [ @@ -99,7 +99,7 @@ for x in [ try: r = sess.run(None, {input_name: x}) print(f"Shape={x.shape} and predicted probabilities={r[1]}") - except (RuntimeError, InvalidArgument) as e: # noqa: PERF203 + except (RuntimeError, InvalidArgument) as e: print(f"ERROR with Shape={x.shape} - {e}") ######################### @@ -114,5 +114,5 @@ for x in [ try: r = sess.run([output_name], {input_name: x}) print(f"Shape={x.shape} and predicted labels={r}") - except (RuntimeError, InvalidArgument) as e: # noqa: PERF203 + except (RuntimeError, InvalidArgument) as e: print(f"ERROR with Shape={x.shape} - {e}") diff --git a/onnxruntime/python/backend/backend.py b/onnxruntime/python/backend/backend.py index 9d16e9cb09..1edae383e9 100644 --- a/onnxruntime/python/backend/backend.py +++ b/onnxruntime/python/backend/backend.py @@ -66,7 +66,7 @@ class OnnxRuntimeBackend(Backend): " Got Domain '{}' version '{}'.".format(domain, opset.version) ) return False, error_message - except AttributeError: # noqa: PERF203 + except AttributeError: # for some CI pipelines accessing helper.OP_SET_ID_VERSION_MAP # is generating attribute error. TODO investigate the pipelines to # fix this error. Falling back to a simple version check when this error is encountered diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index ce408a2ce3..88680cbe57 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -191,7 +191,7 @@ class Session: missing_input_names = [] for input in self._inputs_meta: if input.name not in feed_input_names and not input.type.startswith("optional"): - missing_input_names.append(input.name) # noqa: PERF401 + missing_input_names.append(input.name) if missing_input_names: raise ValueError( f"Required inputs ({missing_input_names}) are missing from input feed ({feed_input_names})." diff --git a/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py b/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py index b7e398e7f7..e0967ef554 100644 --- a/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py +++ b/onnxruntime/python/tools/custom_op_wrapper/create_custom_op_wrapper.py @@ -96,7 +96,7 @@ class ParseIOInfoAction(argparse.Action): try: comp_strs = io_str.split(";") - except ValueError: # noqa: PERF203 + except ValueError: parser.error(f"{opt_str}: {io_meta_name} info must be separated by ';'") if len(comp_strs) != 3: diff --git a/onnxruntime/python/tools/kernel_explorer/kernels/batched_gemm_test.py b/onnxruntime/python/tools/kernel_explorer/kernels/batched_gemm_test.py index 71971a0c86..73323d767a 100644 --- a/onnxruntime/python/tools/kernel_explorer/kernels/batched_gemm_test.py +++ b/onnxruntime/python/tools/kernel_explorer/kernels/batched_gemm_test.py @@ -78,7 +78,7 @@ def _test_batched_gemm( for i in range(batch): try: np.testing.assert_allclose(my_cs[i], ref_cs[i], rtol=bounds[i]) - except Exception as err: # noqa: PERF203 + except Exception as err: header = "*" * 30 + impl + "*" * 30 print(header, bounds[i]) print(err) diff --git a/onnxruntime/python/tools/kernel_explorer/kernels/gemm_softmax_gemm_permute_test.py b/onnxruntime/python/tools/kernel_explorer/kernels/gemm_softmax_gemm_permute_test.py index 0ff9f775c2..64c7c76a1a 100644 --- a/onnxruntime/python/tools/kernel_explorer/kernels/gemm_softmax_gemm_permute_test.py +++ b/onnxruntime/python/tools/kernel_explorer/kernels/gemm_softmax_gemm_permute_test.py @@ -182,7 +182,7 @@ def _test_gemm_softmax_gemm_permute( is_zero_tol, atol, rtol = 1e-3, 2e-2, 1e-2 not_close_to_zeros = np.abs(ref) > is_zero_tol np.testing.assert_allclose(out[not_close_to_zeros], ref[not_close_to_zeros], atol=atol, rtol=rtol) - except Exception as err: # noqa: PERF203 + except Exception as err: header = "*" * 30 + impl + "*" * 30 print(header) print(err) diff --git a/onnxruntime/python/tools/kernel_explorer/kernels/gemm_test.py b/onnxruntime/python/tools/kernel_explorer/kernels/gemm_test.py index 0dfaa05976..6cb984935c 100644 --- a/onnxruntime/python/tools/kernel_explorer/kernels/gemm_test.py +++ b/onnxruntime/python/tools/kernel_explorer/kernels/gemm_test.py @@ -58,7 +58,7 @@ def _test_gemm(func, dtype: str, transa: bool, transb: bool, m: int, n: int, k: try: np.testing.assert_allclose(my_c, ref_c, rtol=bound) - except Exception as err: # noqa: PERF203 + except Exception as err: header = "*" * 30 + impl + "*" * 30 print(header) print(err) diff --git a/onnxruntime/python/tools/kernel_explorer/kernels/strided_batched_gemm_test.py b/onnxruntime/python/tools/kernel_explorer/kernels/strided_batched_gemm_test.py index 1021b36959..9b2b0b0871 100644 --- a/onnxruntime/python/tools/kernel_explorer/kernels/strided_batched_gemm_test.py +++ b/onnxruntime/python/tools/kernel_explorer/kernels/strided_batched_gemm_test.py @@ -82,7 +82,7 @@ def _test_strided_batched_gemm( for i in range(batch): try: np.testing.assert_allclose(my_c[i], ref_c[i], rtol=bounds[i]) - except Exception as err: # noqa: PERF203 + except Exception as err: header = "*" * 30 + impl + "*" * 30 print(header, bounds[i]) print(err) diff --git a/onnxruntime/python/tools/pytorch_export_contrib_ops.py b/onnxruntime/python/tools/pytorch_export_contrib_ops.py index ee86adb766..aeb78f03dd 100644 --- a/onnxruntime/python/tools/pytorch_export_contrib_ops.py +++ b/onnxruntime/python/tools/pytorch_export_contrib_ops.py @@ -96,7 +96,7 @@ def unregister(): for name in _registered_ops: try: torch.onnx.unregister_custom_op_symbolic(name, _OPSET_VERSION) - except AttributeError: # noqa: PERF203 + except AttributeError: # The symbolic_registry module was removed in PyTorch 1.13. # We are importing it here for backwards compatibility # because unregister_custom_op_symbolic is not available before PyTorch 1.12 diff --git a/onnxruntime/python/tools/quantization/calibrate.py b/onnxruntime/python/tools/quantization/calibrate.py index 064be434c8..6949d2dac3 100644 --- a/onnxruntime/python/tools/quantization/calibrate.py +++ b/onnxruntime/python/tools/quantization/calibrate.py @@ -370,7 +370,7 @@ class HistogramCalibrater(CalibraterBase): self.tensors_to_calibrate, value_infos = self.select_tensors_to_calibrate(self.model) for tensor in self.tensors_to_calibrate: if tensor not in self.model_original_outputs: - self.model.graph.output.append(value_infos[tensor]) # noqa: PERF401 + self.model.graph.output.append(value_infos[tensor]) onnx.save( self.model, diff --git a/onnxruntime/python/tools/quantization/onnx_model.py b/onnxruntime/python/tools/quantization/onnx_model.py index d2e16f50e6..1498e43a08 100644 --- a/onnxruntime/python/tools/quantization/onnx_model.py +++ b/onnxruntime/python/tools/quantization/onnx_model.py @@ -186,7 +186,7 @@ class ONNXModel: parents = [] for input in node.input: if input in output_name_to_node: - parents.append(output_name_to_node[input]) # noqa: PERF401 + parents.append(output_name_to_node[input]) return parents def get_parent(self, node, idx, output_name_to_node=None): @@ -222,7 +222,7 @@ class ONNXModel: for node in graph.node: for node_input in node.input: if node_input == initializer.name: - nodes.append(node) # noqa: PERF401 + nodes.append(node) return nodes @staticmethod @@ -379,7 +379,7 @@ class ONNXModel: and not self.is_graph_output(node.output[0]) and node.output[0] not in input_name_to_nodes ): - unused_nodes.append(node) # noqa: PERF401 + unused_nodes.append(node) self.remove_nodes(unused_nodes) diff --git a/onnxruntime/python/tools/quantization/qdq_loss_debug.py b/onnxruntime/python/tools/quantization/qdq_loss_debug.py index dd4e831b4b..67938de54a 100644 --- a/onnxruntime/python/tools/quantization/qdq_loss_debug.py +++ b/onnxruntime/python/tools/quantization/qdq_loss_debug.py @@ -145,7 +145,7 @@ def collect_activations( intermediate_outputs = [] for input_d in input_reader: - intermediate_outputs.append(inference_session.run(None, input_d)) # noqa: PERF401 + intermediate_outputs.append(inference_session.run(None, input_d)) if not intermediate_outputs: raise RuntimeError("No data is collected while running augmented model!") diff --git a/onnxruntime/python/tools/symbolic_shape_infer.py b/onnxruntime/python/tools/symbolic_shape_infer.py index 1e818ef7f7..138a50cbe2 100755 --- a/onnxruntime/python/tools/symbolic_shape_infer.py +++ b/onnxruntime/python/tools/symbolic_shape_infer.py @@ -1008,13 +1008,13 @@ class SymbolicShapeInference: right_ellipsis_index = right_equation.find(b"...") if right_ellipsis_index != -1: for i in range(num_ellipsis_indices): - new_sympy_shape.append(shape[i]) # noqa: PERF401 + new_sympy_shape.append(shape[i]) for c in right_equation: if c != 46: # c != b'.' - new_sympy_shape.append(letter_to_dim[c]) # noqa: PERF401 + new_sympy_shape.append(letter_to_dim[c]) else: for i in range(num_ellipsis_indices): - new_sympy_shape.append(shape[i]) # noqa: PERF401 + new_sympy_shape.append(shape[i]) for c in left_equation: if c != 44 and c != 46: # c != b',' and c != b'.': if c in num_letter_occurrences: diff --git a/onnxruntime/python/tools/tensorrt/perf/benchmark.py b/onnxruntime/python/tools/tensorrt/perf/benchmark.py index 7cf0839024..d440cafb23 100644 --- a/onnxruntime/python/tools/tensorrt/perf/benchmark.py +++ b/onnxruntime/python/tools/tensorrt/perf/benchmark.py @@ -138,7 +138,7 @@ def run_trt_standalone(trtexec, model_name, model_path, test_data_dir, all_input logger.info(loaded_input) shape = [] for j in all_inputs_shape[i]: - shape.append(str(j)) # noqa: PERF401 + shape.append(str(j)) shape = "x".join(shape) shape = name + ":" + shape input_shape.append(shape) @@ -266,7 +266,7 @@ def get_ort_session_inputs_and_outputs(name, session, ort_input): for i in range(len(session.get_inputs())): sess_inputs[session.get_inputs()[i].name] = ort_input[i] for i in range(len(session.get_outputs())): - sess_outputs.append(session.get_outputs()[i].name) # noqa: PERF401 + sess_outputs.append(session.get_outputs()[i].name) return (sess_inputs, sess_outputs) @@ -406,7 +406,7 @@ def inference_ort( runtime = runtime[1:] # remove warmup runtimes += runtime - except Exception as e: # noqa: PERF203 + except Exception as e: logger.error(e) if track_memory: end_memory_tracking(p, success) @@ -605,7 +605,7 @@ def validate(all_ref_outputs, all_outputs, rtol, atol, percent_mismatch): # abs(desired-actual) < rtol * abs(desired) + atol try: np.testing.assert_allclose(ref_o, o, rtol, atol) - except Exception as e: # noqa: PERF203 + except Exception as e: if percentage_in_allowed_threshold(e, percent_mismatch): continue logger.error(e) @@ -2051,7 +2051,7 @@ class ParseDictArgAction(argparse.Action): for kv in values.split(","): try: k, v = kv.split("=") - except ValueError: # noqa: PERF203 + except ValueError: parser.error(f"argument {option_string}: Expected '=' between key and value") if k in dict_arg: diff --git a/onnxruntime/python/tools/tensorrt/perf/post.py b/onnxruntime/python/tools/tensorrt/perf/post.py index dffe270b18..350e8b3914 100644 --- a/onnxruntime/python/tools/tensorrt/perf/post.py +++ b/onnxruntime/python/tools/tensorrt/perf/post.py @@ -146,7 +146,7 @@ def get_memory(memory, model_group): memory_columns = [model_title] for provider in provider_list: if cpu not in provider: - memory_columns.append(provider + memory_ending) # noqa: PERF401 + memory_columns.append(provider + memory_ending) memory_db_columns = [ model_title, cuda, @@ -273,7 +273,7 @@ def get_latency(latency, model_group): latency_columns = [model_title] for provider in provider_list: - latency_columns.append(provider + avg_ending) # noqa: PERF401 + latency_columns.append(provider + avg_ending) latency_db_columns = table_headers latency = adjust_columns(latency, latency_columns, latency_db_columns, model_group) return latency diff --git a/onnxruntime/python/tools/tensorrt/perf/setup_scripts/setup_onnx_zoo.py b/onnxruntime/python/tools/tensorrt/perf/setup_scripts/setup_onnx_zoo.py index b36cd67878..4f763ad844 100644 --- a/onnxruntime/python/tools/tensorrt/perf/setup_scripts/setup_onnx_zoo.py +++ b/onnxruntime/python/tools/tensorrt/perf/setup_scripts/setup_onnx_zoo.py @@ -75,7 +75,7 @@ def main(): model_list = [] for link in links: - model_list.append(get_model_info(link)) # noqa: PERF401 + model_list.append(get_model_info(link)) write_json(model_list) diff --git a/onnxruntime/python/tools/transformers/benchmark.py b/onnxruntime/python/tools/transformers/benchmark.py index 023f4a7414..bd9a649ae7 100644 --- a/onnxruntime/python/tools/transformers/benchmark.py +++ b/onnxruntime/python/tools/transformers/benchmark.py @@ -417,7 +417,7 @@ def run_pytorch( result.update(get_latency_result(runtimes, batch_size)) logger.info(result) results.append(result) - except RuntimeError as e: # noqa: PERF203 + except RuntimeError as e: logger.exception(e) torch.cuda.empty_cache() @@ -572,7 +572,7 @@ def run_tensorflow( result.update(get_latency_result(runtimes, batch_size)) logger.info(result) results.append(result) - except RuntimeError as e: # noqa: PERF203 + except RuntimeError as e: logger.exception(e) from numba import cuda diff --git a/onnxruntime/python/tools/transformers/benchmark_helper.py b/onnxruntime/python/tools/transformers/benchmark_helper.py index 639b2f3462..5fa64d1bc0 100644 --- a/onnxruntime/python/tools/transformers/benchmark_helper.py +++ b/onnxruntime/python/tools/transformers/benchmark_helper.py @@ -249,7 +249,7 @@ def output_summary(results, csv_filename, args): data_names.append(f"b{batch_size}") else: for sequence_length in args.sequence_lengths: - data_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401 + data_names.append(f"b{batch_size}_s{sequence_length}") csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + data_names) csv_writer.writeheader() @@ -386,7 +386,7 @@ def allocateOutputBuffers(output_buffers, output_buffer_max_sizes, device): # n # for each test run. for i in output_buffer_max_sizes: - output_buffers.append(torch.empty(i, dtype=torch.float32, device=device)) # noqa: PERF401 + output_buffers.append(torch.empty(i, dtype=torch.float32, device=device)) def set_random_seed(seed=123): diff --git a/onnxruntime/python/tools/transformers/convert_tf_models_to_pytorch.py b/onnxruntime/python/tools/transformers/convert_tf_models_to_pytorch.py index 89ed140e81..1027cd7213 100644 --- a/onnxruntime/python/tools/transformers/convert_tf_models_to_pytorch.py +++ b/onnxruntime/python/tools/transformers/convert_tf_models_to_pytorch.py @@ -197,7 +197,7 @@ def tf2pt_pipeline_test(): input = torch.randint(low=0, high=config.vocab_size - 1, size=(4, 128), dtype=torch.long) try: model(input) - except RuntimeError as e: # noqa: PERF203 + except RuntimeError as e: logger.exception(e) diff --git a/onnxruntime/python/tools/transformers/convert_to_packing_mode.py b/onnxruntime/python/tools/transformers/convert_to_packing_mode.py index 5c49d80d64..f5ec5b884f 100644 --- a/onnxruntime/python/tools/transformers/convert_to_packing_mode.py +++ b/onnxruntime/python/tools/transformers/convert_to_packing_mode.py @@ -124,7 +124,7 @@ class PackingMode: attributes = [] for attr in attention.attribute: if attr.name in ["num_heads", "qkv_hidden_sizes", "scale"]: - attributes.append(attr) # noqa: PERF401 + attributes.append(attr) packed_attention.attribute.extend(attributes) packed_attention.domain = "com.microsoft" diff --git a/onnxruntime/python/tools/transformers/models/bert/eval_squad.py b/onnxruntime/python/tools/transformers/models/bert/eval_squad.py index c7194b377a..66265d7b1e 100644 --- a/onnxruntime/python/tools/transformers/models/bert/eval_squad.py +++ b/onnxruntime/python/tools/transformers/models/bert/eval_squad.py @@ -150,7 +150,7 @@ def output_summary(results: List[Dict[str, Any]], csv_filename: str, metric_name key_names = [] for sequence_length in sequence_lengths: for batch_size in batch_sizes: - key_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401 + key_names.append(f"b{batch_size}_s{sequence_length}") csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + key_names) csv_writer.writeheader() diff --git a/onnxruntime/python/tools/transformers/models/gpt2/benchmark_gpt2.py b/onnxruntime/python/tools/transformers/models/gpt2/benchmark_gpt2.py index 20160d0406..e8553e2cae 100644 --- a/onnxruntime/python/tools/transformers/models/gpt2/benchmark_gpt2.py +++ b/onnxruntime/python/tools/transformers/models/gpt2/benchmark_gpt2.py @@ -364,7 +364,7 @@ def main(args): # Results of IO binding might be in GPU. Copy outputs to CPU for comparison. copy_outputs = [] for output in ort_outputs: - copy_outputs.append(output.cpu().numpy()) # noqa: PERF401 + copy_outputs.append(output.cpu().numpy()) if gpt2helper.compare_outputs( outputs, @@ -404,7 +404,7 @@ def main(args): "onnxruntime_latency": f"{ort_latency:.2f}", } csv_writer.writerow(row) - except Exception: # noqa: PERF203 + except Exception: logger.error("Exception", exc_info=True) return None diff --git a/onnxruntime/python/tools/transformers/models/gpt2/gpt2_helper.py b/onnxruntime/python/tools/transformers/models/gpt2/gpt2_helper.py index 50cdc92e61..b10f5ba763 100644 --- a/onnxruntime/python/tools/transformers/models/gpt2/gpt2_helper.py +++ b/onnxruntime/python/tools/transformers/models/gpt2/gpt2_helper.py @@ -75,7 +75,7 @@ class MyGPT2Model(GPT2Model): for i in range(num_layer): # Since transformers v4.*, past key and values are separated outputs. # Here we concate them into one tensor to be compatible with Attention operator. - present.append( # noqa: PERF401 + present.append( torch.cat( (result[1][i][0].unsqueeze(0), result[1][i][1].unsqueeze(0)), dim=0, diff --git a/onnxruntime/python/tools/transformers/models/gpt2/gpt2_parity.py b/onnxruntime/python/tools/transformers/models/gpt2/gpt2_parity.py index f70b6520e9..905c56ff93 100644 --- a/onnxruntime/python/tools/transformers/models/gpt2/gpt2_parity.py +++ b/onnxruntime/python/tools/transformers/models/gpt2/gpt2_parity.py @@ -256,7 +256,7 @@ def run_significance_test(rows, output_csv_path): utest_statistic, utest_pvalue = scipy.stats.mannwhitneyu( a, b, use_continuity=True, alternative="two-sided" ) # TODO: shall we use one-sided: less or greater according to "top1_match_rate" - except ValueError: # ValueError: All numbers are identical in mannwhitneyu # noqa: PERF203 + except ValueError: # ValueError: All numbers are identical in mannwhitneyu utest_statistic = None utest_pvalue = None ttest_statistic, ttest_pvalue = scipy.stats.ttest_ind(a, b, axis=None, equal_var=True) diff --git a/onnxruntime/python/tools/transformers/models/longformer/benchmark_longformer.py b/onnxruntime/python/tools/transformers/models/longformer/benchmark_longformer.py index 5f67498065..bf6c1e6030 100644 --- a/onnxruntime/python/tools/transformers/models/longformer/benchmark_longformer.py +++ b/onnxruntime/python/tools/transformers/models/longformer/benchmark_longformer.py @@ -645,7 +645,7 @@ def run_tests( args = parse_arguments(f"{arguments} -t {test_times}".split(" ")) latency_results = launch_test(args) - except KeyboardInterrupt as exc: # noqa: PERF203 + except KeyboardInterrupt as exc: raise RuntimeError("Keyboard Interrupted") from exc except Exception: traceback.print_exc() @@ -687,7 +687,7 @@ def output_summary(results, csv_filename, data_field="average_latency_ms"): data_names = [] for sequence_length in sequence_lengths: for batch_size in batch_sizes: - data_names.append(f"b{batch_size}_s{sequence_length}") # noqa: PERF401 + data_names.append(f"b{batch_size}_s{sequence_length}") csv_writer = csv.DictWriter(csv_file, fieldnames=header_names + data_names) csv_writer.writeheader() diff --git a/onnxruntime/python/tools/transformers/models/t5/t5_decoder.py b/onnxruntime/python/tools/transformers/models/t5/t5_decoder.py index ceeb96e877..0b8f4919b8 100644 --- a/onnxruntime/python/tools/transformers/models/t5/t5_decoder.py +++ b/onnxruntime/python/tools/transformers/models/t5/t5_decoder.py @@ -204,10 +204,10 @@ class T5DecoderInputs: past = [] for _ in range(2 * num_layers): - past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401 + past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) for _ in range(2 * num_layers): - past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401 + past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) else: past = None diff --git a/onnxruntime/python/tools/transformers/models/whisper/whisper_decoder.py b/onnxruntime/python/tools/transformers/models/whisper/whisper_decoder.py index e85757ded8..d5cef8e3b1 100644 --- a/onnxruntime/python/tools/transformers/models/whisper/whisper_decoder.py +++ b/onnxruntime/python/tools/transformers/models/whisper/whisper_decoder.py @@ -167,10 +167,10 @@ class WhisperDecoderInputs: past = [] for _ in range(2 * num_layers): - past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401 + past.append(torch.rand(self_attention_past_shape, dtype=float_type, device=device)) for _ in range(2 * num_layers): - past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) # noqa: PERF401 + past.append(torch.rand(cross_attention_past_shape, dtype=float_type, device=device)) else: past = None diff --git a/onnxruntime/python/tools/transformers/onnx_model.py b/onnxruntime/python/tools/transformers/onnx_model.py index fe6b877e56..7d3b363650 100644 --- a/onnxruntime/python/tools/transformers/onnx_model.py +++ b/onnxruntime/python/tools/transformers/onnx_model.py @@ -108,14 +108,14 @@ class OnnxModel: input_names = [] for graph in self.graphs(): for input in graph.input: - input_names.append(input.name) # noqa: PERF401 + input_names.append(input.name) return input_names def get_graphs_output_names(self): output_names = [] for graph in self.graphs(): for output in graph.output: - output_names.append(output.name) # noqa: PERF401 + output_names.append(output.name) return output_names def get_graph_by_node(self, node): @@ -217,7 +217,7 @@ class OnnxModel: nodes = [] for node in self.nodes(): if node.op_type == op_type: - nodes.append(node) # noqa: PERF401 + nodes.append(node) return nodes def get_children(self, node, input_name_to_nodes=None): @@ -238,7 +238,7 @@ class OnnxModel: parents = [] for input in node.input: if input in output_name_to_node: - parents.append(output_name_to_node[input]) # noqa: PERF401 + parents.append(output_name_to_node[input]) return parents def get_parent(self, node, i, output_name_to_node=None): @@ -792,7 +792,7 @@ class OnnxModel: nodes = self.nodes() for node in nodes: if node.op_type == "Constant" and node.output[0] not in input_name_to_nodes: - unused_nodes.append(node) # noqa: PERF401 + unused_nodes.append(node) self.remove_nodes(unused_nodes) @@ -837,7 +837,7 @@ class OnnxModel: output_to_remove = [] for output in self.model.graph.output: if output.name not in outputs: - output_to_remove.append(output) # noqa: PERF401 + output_to_remove.append(output) for output in output_to_remove: self.model.graph.output.remove(output) @@ -882,7 +882,7 @@ class OnnxModel: if allow_remove_graph_inputs: for input in graph.input: if input.name not in remaining_input_names: - inputs_to_remove.append(input) # noqa: PERF401 + inputs_to_remove.append(input) for input in inputs_to_remove: graph.input.remove(input) @@ -1058,7 +1058,7 @@ class OnnxModel: graph_inputs = [] for input in self.model.graph.input: if self.get_initializer(input.name) is None: - graph_inputs.append(input) # noqa: PERF401 + graph_inputs.append(input) return graph_inputs def get_opset_version(self): diff --git a/onnxruntime/python/tools/transformers/onnx_model_unet.py b/onnxruntime/python/tools/transformers/onnx_model_unet.py index 09a6ecea9f..00fc0763d8 100644 --- a/onnxruntime/python/tools/transformers/onnx_model_unet.py +++ b/onnxruntime/python/tools/transformers/onnx_model_unet.py @@ -47,7 +47,7 @@ class UnetOnnxModel(BertOnnxModel): nodes_to_remove = [] for div in div_nodes: if self.find_constant_input(div, 1.0) == 1: - nodes_to_remove.append(div) # noqa: PERF401 + nodes_to_remove.append(div) for node in nodes_to_remove: self.replace_input_of_all_nodes(node.output[0], node.input[0]) diff --git a/onnxruntime/python/tools/transformers/shape_optimizer.py b/onnxruntime/python/tools/transformers/shape_optimizer.py index c3fa0435dc..ac62188662 100644 --- a/onnxruntime/python/tools/transformers/shape_optimizer.py +++ b/onnxruntime/python/tools/transformers/shape_optimizer.py @@ -78,7 +78,7 @@ class BertOnnxModelShapeOptimizer(OnnxModel): shape_inputs = [] for node in self.model.graph.node: if node.op_type == "Reshape": - shape_inputs.append(node.input[1]) # noqa: PERF401 + shape_inputs.append(node.input[1]) return shape_inputs diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 096f6c8004..988b5dc07c 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -406,7 +406,7 @@ class TestInferenceSession(unittest.TestCase): run_base_test2() run_advanced_test() - except OSError: # noqa: PERF203 + except OSError: continue else: break diff --git a/onnxruntime/test/python/quantization/test_calibration.py b/onnxruntime/test/python/quantization/test_calibration.py index 93c684cbc2..14be6fa45c 100644 --- a/onnxruntime/test/python/quantization/test_calibration.py +++ b/onnxruntime/test/python/quantization/test_calibration.py @@ -35,7 +35,7 @@ class TestDataReader(CalibrationDataReader): self.count = 4 self.input_data_list = [] for _ in range(self.count): - self.input_data_list.append(np.random.normal(0, 0.33, [1, 3, 1, 3]).astype(np.float32)) # noqa: PERF401 + self.input_data_list.append(np.random.normal(0, 0.33, [1, 3, 1, 3]).astype(np.float32)) def get_next(self): if self.preprocess_flag: diff --git a/onnxruntime/test/python/quantization/test_qdq_loss_debug.py b/onnxruntime/test/python/quantization/test_qdq_loss_debug.py index 3087cb9633..e9108f157f 100644 --- a/onnxruntime/test/python/quantization/test_qdq_loss_debug.py +++ b/onnxruntime/test/python/quantization/test_qdq_loss_debug.py @@ -93,7 +93,7 @@ class TestDataReader(CalibrationDataReader): self.count = 2 self.input_data_list = [] for _ in range(self.count): - self.input_data_list.append(np.random.normal(0, 0.33, input_shape).astype(np.float32)) # noqa: PERF401 + self.input_data_list.append(np.random.normal(0, 0.33, input_shape).astype(np.float32)) def get_next(self): if self.preprocess_flag: @@ -144,7 +144,7 @@ class TestSaveActivations(unittest.TestCase): data_reader.rewind() oracle_outputs = [] for input_d in data_reader: - oracle_outputs.append(infer_session.run(None, input_d)) # noqa: PERF401 + oracle_outputs.append(infer_session.run(None, input_d)) output_dict = {} output_info = infer_session.get_outputs() diff --git a/onnxruntime/test/python/transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/generate_tiny_keras2onnx_bert_models.py b/onnxruntime/test/python/transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/generate_tiny_keras2onnx_bert_models.py index 9986e81cd7..c42c42c3ca 100644 --- a/onnxruntime/test/python/transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/generate_tiny_keras2onnx_bert_models.py +++ b/onnxruntime/test/python/transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/generate_tiny_keras2onnx_bert_models.py @@ -342,7 +342,7 @@ def generate_test_data( path = os.path.join(output_path, "test_data_set_" + str(test_case)) try: os.mkdir(path) - except OSError: # noqa: PERF203 + except OSError: print("Creation of the directory %s failed" % path) else: print("Successfully created the directory %s " % path) diff --git a/onnxruntime/test/python/transformers/test_data/gpt2_pytorch1.5_opset11/generate_tiny_gpt2_model.py b/onnxruntime/test/python/transformers/test_data/gpt2_pytorch1.5_opset11/generate_tiny_gpt2_model.py index 3e088f3870..065783d581 100644 --- a/onnxruntime/test/python/transformers/test_data/gpt2_pytorch1.5_opset11/generate_tiny_gpt2_model.py +++ b/onnxruntime/test/python/transformers/test_data/gpt2_pytorch1.5_opset11/generate_tiny_gpt2_model.py @@ -451,7 +451,7 @@ def generate_test_data( path = os.path.join(output_path, "test_data_set_" + str(test_case)) try: os.mkdir(path) - except OSError: # noqa: PERF203 + except OSError: print("Creation of the directory %s failed" % path) else: print("Successfully created the directory %s " % path) diff --git a/orttraining/orttraining/python/training/checkpoint.py b/orttraining/orttraining/python/training/checkpoint.py index 079c827ea7..d0ff065066 100644 --- a/orttraining/orttraining/python/training/checkpoint.py +++ b/orttraining/orttraining/python/training/checkpoint.py @@ -145,7 +145,7 @@ def _order_paths(paths, D_groups, H_groups): world_rank = _utils.state_dict_trainer_options_world_rank_key() for path in paths: - trainer_options_path_tuples.append( # noqa: PERF401 + trainer_options_path_tuples.append( (_checkpoint_storage.load(path, key=_utils.state_dict_trainer_options_key()), path) ) @@ -365,7 +365,7 @@ def _get_parallellism_groups(data_parallel_size, horizontal_parallel_size, world for data_group_id in range(num_data_groups): data_group_ranks = [] for r in range(data_parallel_size): - data_group_ranks.append(data_group_id + horizontal_parallel_size * r) # noqa: PERF401 + data_group_ranks.append(data_group_id + horizontal_parallel_size * r) data_groups.append(data_group_ranks) num_horizontal_groups = world_size // horizontal_parallel_size @@ -373,7 +373,7 @@ def _get_parallellism_groups(data_parallel_size, horizontal_parallel_size, world for hori_group_id in range(num_horizontal_groups): hori_group_ranks = [] for r in range(horizontal_parallel_size): - hori_group_ranks.append(hori_group_id * horizontal_parallel_size + r) # noqa: PERF401 + hori_group_ranks.append(hori_group_id * horizontal_parallel_size + r) horizontal_groups.append(hori_group_ranks) return data_groups, horizontal_groups diff --git a/orttraining/orttraining/python/training/onnxblock/optim/optim.py b/orttraining/orttraining/python/training/onnxblock/optim/optim.py index 8a5e387342..94d4c2791d 100644 --- a/orttraining/orttraining/python/training/onnxblock/optim/optim.py +++ b/orttraining/orttraining/python/training/onnxblock/optim/optim.py @@ -187,7 +187,7 @@ class AdamW(onnxblock_module.ForwardBlock): # Prepare the tensor sequence inputs for params and moments for input_name in [params_name, gradients_name, first_order_moments_name, second_order_moments_name]: - onnx_model.graph.input.append( # noqa: PERF401 + onnx_model.graph.input.append( onnx.helper.make_tensor_sequence_value_info(input_name, trainable_parameters[0].data_type, None) ) diff --git a/orttraining/orttraining/python/training/ort_triton/_lowering.py b/orttraining/orttraining/python/training/ort_triton/_lowering.py index dacd5c2cac..16db9ab000 100644 --- a/orttraining/orttraining/python/training/ort_triton/_lowering.py +++ b/orttraining/orttraining/python/training/ort_triton/_lowering.py @@ -294,7 +294,7 @@ class GraphLowering: producers[output] = node for input in node.input: if input in producers: - precessors[node.name].append(producers[input]) # noqa: PERF401 + precessors[node.name].append(producers[input]) for value in precessors.values(): value.sort(key=sorted_nodes.index, reverse=True) for idx in range(len(sorted_nodes) - 1, -1, -1): @@ -441,9 +441,7 @@ class GraphLowering: assert isinstance(sub_nodes[nxt], ReduceForLoopEnd) for reduce_node in sub_nodes[nxt].reduce_nodes: if reduce_node.outputs[0].name in output_name_map: - reduce_store_nodes.append( # noqa: PERF401 - IONode(reduce_node.outputs[0], kernel_node.offset_calc, False) - ) + reduce_store_nodes.append(IONode(reduce_node.outputs[0], kernel_node.offset_calc, False)) new_sub_nodes.append(sub_nodes[nxt]) nxt += 1 cur = nxt diff --git a/orttraining/orttraining/python/training/ort_triton/_sorted_graph.py b/orttraining/orttraining/python/training/ort_triton/_sorted_graph.py index 3c34e65f8c..69df567500 100644 --- a/orttraining/orttraining/python/training/ort_triton/_sorted_graph.py +++ b/orttraining/orttraining/python/training/ort_triton/_sorted_graph.py @@ -110,7 +110,7 @@ class SortedGraph: for node_idx, node in enumerate(self._sorted_nodes): inputs = [] for input in node.input: - inputs.append(name_map.get(input, input)) # noqa: PERF401 + inputs.append(name_map.get(input, input)) inputs_str = ",".join(inputs) outputs = [] for idx, output in enumerate(node.output): @@ -180,7 +180,7 @@ class SortedGraph: else: input_infos = [] for input in node.input: - input_infos.append(self._node_arg_infos[input]) # noqa: PERF401 + input_infos.append(self._node_arg_infos[input]) output_infos = TypeAndShapeInfer.infer(node, input_infos, self._graph) for idx, output in enumerate(node.output): self._node_arg_infos[output] = output_infos[idx] diff --git a/orttraining/orttraining/python/training/ort_triton/_utils.py b/orttraining/orttraining/python/training/ort_triton/_utils.py index 35dba06c76..c80e28f6f7 100644 --- a/orttraining/orttraining/python/training/ort_triton/_utils.py +++ b/orttraining/orttraining/python/training/ort_triton/_utils.py @@ -52,7 +52,7 @@ def topological_sort(inputs: List[str], nodes: List[NodeProto]) -> List[NodeProt continue for consumer in non_const_nodes: if output in consumer.input: - output_consumers[node.name].append(consumer) # noqa: PERF401 + output_consumers[node.name].append(consumer) # Topological sort. visited = set() diff --git a/orttraining/orttraining/python/training/ort_triton/kernel/_slice_scel.py b/orttraining/orttraining/python/training/ort_triton/kernel/_slice_scel.py index 446344a7cd..8edcc9b63e 100644 --- a/orttraining/orttraining/python/training/ort_triton/kernel/_slice_scel.py +++ b/orttraining/orttraining/python/training/ort_triton/kernel/_slice_scel.py @@ -357,7 +357,7 @@ def transform_slice_scel(graph): all_nodes = [] for node in graph.node: if node not in remove_nodes: - all_nodes.append(node) # noqa: PERF401 + all_nodes.append(node) for node in triton_nodes: all_nodes.append(node) # noqa: PERF402 diff --git a/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py b/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py index 1061135388..dcaa202d46 100644 --- a/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py +++ b/orttraining/orttraining/python/training/ortmodule/experimental/hierarchical_ortmodule/_hierarchical_ortmodule.py @@ -36,7 +36,7 @@ class _IteratedORTModule(torch.nn.Module): self._it = count - 1 self._ortmodules = [] for idx in range(count): - self._ortmodules.append( # noqa: PERF401 + self._ortmodules.append( ORTModule( module, debug_options=DebugOptions( diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py index 225a01c39f..bb0952dea5 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py @@ -20,7 +20,7 @@ def _list_extensions(path): for root, _, files in os.walk(path): for name in files: if name.lower() == "setup.py": - extensions.append(os.path.join(root, name)) # noqa: PERF401 + extensions.append(os.path.join(root, name)) return extensions diff --git a/orttraining/orttraining/python/training/orttrainer.py b/orttraining/orttraining/python/training/orttrainer.py index 3a9dbc0846..a6c6c8af27 100644 --- a/orttraining/orttraining/python/training/orttrainer.py +++ b/orttraining/orttraining/python/training/orttrainer.py @@ -933,7 +933,7 @@ class ORTTrainer: # so output will be on the same device as input. try: torch.device(target_device) - except Exception: # noqa: PERF203 + except Exception: # in this case, input/output must on CPU assert input.device.type == "cpu" target_device = "cpu" diff --git a/orttraining/orttraining/python/training/postprocess.py b/orttraining/orttraining/python/training/postprocess.py index aafc6afce2..6c2adb6af7 100644 --- a/orttraining/orttraining/python/training/postprocess.py +++ b/orttraining/orttraining/python/training/postprocess.py @@ -26,7 +26,7 @@ def find_input_node(model, arg): for node in model.graph.node: for output in node.output: if output == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -35,7 +35,7 @@ def find_output_node(model, arg): for node in model.graph.node: for input in node.input: if input == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else result @@ -189,7 +189,7 @@ def find_nodes(graph, op_type): nodes = [] for node in graph.node: if node.op_type == op_type: - nodes.append(node) # noqa: PERF401 + nodes.append(node) return nodes @@ -382,7 +382,7 @@ def layer_norm_transform(model): all_nodes = [] for node in graph.node: if node not in remove_nodes: - all_nodes.append(node) # noqa: PERF401 + all_nodes.append(node) for node in layer_norm_nodes: all_nodes.append(node) # noqa: PERF402 diff --git a/orttraining/orttraining/test/python/onnxruntime_test_postprocess.py b/orttraining/orttraining/test/python/onnxruntime_test_postprocess.py index 380db8cdab..d5298cf8e8 100644 --- a/orttraining/orttraining/test/python/onnxruntime_test_postprocess.py +++ b/orttraining/orttraining/test/python/onnxruntime_test_postprocess.py @@ -64,7 +64,7 @@ class Test_PostPasses(unittest.TestCase): # noqa: N801 nodes = [] for node in model.graph.node: if node.op_type == node_type: - nodes.append(node) # noqa: PERF401 + nodes.append(node) return nodes def get_name(self, name): diff --git a/orttraining/orttraining/test/python/orttraining_test_data_loader.py b/orttraining/orttraining/test/python/orttraining_test_data_loader.py index d55ace62f2..aa15b44ae0 100644 --- a/orttraining/orttraining/test/python/orttraining_test_data_loader.py +++ b/orttraining/orttraining/test/python/orttraining_test_data_loader.py @@ -20,7 +20,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): values = [] for _ in range(total_dims): - values.append(rng.randint(0, vocab_size - 1)) # noqa: PERF401 + values.append(rng.randint(0, vocab_size - 1)) return torch.tensor(data=values, dtype=torch.long).view(shape).contiguous() @@ -36,7 +36,7 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None): values = [] for _ in range(total_dims): - values.append(rng.random() * scale) # noqa: PERF401 + values.append(rng.random() * scale) return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous() diff --git a/orttraining/orttraining/test/python/orttraining_test_hierarchical_ortmodule.py b/orttraining/orttraining/test/python/orttraining_test_hierarchical_ortmodule.py index 9f41927c0e..8afbafccb8 100644 --- a/orttraining/orttraining/test/python/orttraining_test_hierarchical_ortmodule.py +++ b/orttraining/orttraining/test/python/orttraining_test_hierarchical_ortmodule.py @@ -213,7 +213,7 @@ def test_hierarchical_ortmodule(): call_backward(y_ref) g_ref = [] for param in m.parameters(): - g_ref.append(param.grad.detach()) # noqa: PERF401 + g_ref.append(param.grad.detach()) m.zero_grad() @@ -224,7 +224,7 @@ def test_hierarchical_ortmodule(): call_backward(y) g = [] for param in m.parameters(): - g.append(param.grad.detach()) # noqa: PERF401 + g.append(param.grad.detach()) # Some sub-modules become ORTModule. assert expected_num_ortmodule == count_ortmodule(m) diff --git a/orttraining/orttraining/test/python/orttraining_test_layer_norm_transform.py b/orttraining/orttraining/test/python/orttraining_test_layer_norm_transform.py index de8806095f..6a3788e2fc 100644 --- a/orttraining/orttraining/test/python/orttraining_test_layer_norm_transform.py +++ b/orttraining/orttraining/test/python/orttraining_test_layer_norm_transform.py @@ -173,7 +173,7 @@ def layer_norm_transform(model_proto): all_nodes = [] for node in graph_proto.node: if node not in removed_nodes: - all_nodes.append(node) # noqa: PERF401 + all_nodes.append(node) for node in layer_norm_nodes: all_nodes.append(node) # noqa: PERF402 diff --git a/orttraining/orttraining/test/python/orttraining_test_model_transform.py b/orttraining/orttraining/test/python/orttraining_test_model_transform.py index 70694d03f1..3b07aa1f4d 100644 --- a/orttraining/orttraining/test/python/orttraining_test_model_transform.py +++ b/orttraining/orttraining/test/python/orttraining_test_model_transform.py @@ -13,7 +13,7 @@ def find_single_output_node(model, arg): for node in model.graph.node: for input in node.input: if input == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -63,7 +63,7 @@ def fix_transpose(model): for n in model.graph.node: for input in n.input: if input == weight.name: - result.append(n) # noqa: PERF401 + result.append(n) if len(result) > 1: continue perm = node.attribute[0] @@ -93,7 +93,7 @@ def fix_transpose(model): old_ws = [] for t in transpose: if find_single_output_node(model, t[1].name) is None: - old_ws.append(find_weight_index(model, t[1].name)) # noqa: PERF401 + old_ws.append(find_weight_index(model, t[1].name)) old_ws.sort(reverse=True) for w_i in old_ws: del model.graph.initializer[w_i] diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index e67edeecce..625c1ce0d4 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -1497,10 +1497,10 @@ def test_gradient_correctness_einsum(equation): rhs_op = equation[pos1 + 1 : pos2] lhs_shape = [] for c in lhs_op: - lhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401 + lhs_shape.append(SIZE_MAP[c.upper()]) rhs_shape = [] for c in rhs_op: - rhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401 + rhs_shape.append(SIZE_MAP[c.upper()]) pt_model = NeuralNetEinsum(lhs_shape[-1]).to(device) ort_model = ORTModule(copy.deepcopy(pt_model)) @@ -1577,7 +1577,7 @@ def test_gradient_correctness_einsum_2(): random.shuffle(output_candidates) output_candidates = output_candidates[:8] for output_candidate in [list(candidate) for candidate in output_candidates]: - all_cases.append((lhs_candidate, rhs_candidate, output_candidate)) # noqa: PERF401 + all_cases.append((lhs_candidate, rhs_candidate, output_candidate)) for case in all_cases: equation = to_string(case[0]) + "," + to_string(case[1]) + "->" + to_string(case[2]) @@ -1587,10 +1587,10 @@ def test_gradient_correctness_einsum_2(): rhs_op = equation[pos1 + 1 : pos2] lhs_shape = [] for c in lhs_op: - lhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401 + lhs_shape.append(SIZE_MAP[c.upper()]) rhs_shape = [] for c in rhs_op: - rhs_shape.append(SIZE_MAP[c.upper()]) # noqa: PERF401 + rhs_shape.append(SIZE_MAP[c.upper()]) pt_model = NeuralNetEinsum(lhs_shape[-1]).to(device) ort_model = ORTModule(copy.deepcopy(pt_model)) @@ -5895,7 +5895,7 @@ def test_ops_for_padding_elimination(test_cases): result = [] for node in model.graph.node: if arg in node.output: - result.append(node) # noqa: PERF401 + result.append(node) return result[0].op_type if len(result) == 1 else None gathergrad_input_optypes = [find_input_node_type(training_model, arg) for arg in gathergrad_node.input] diff --git a/orttraining/orttraining/test/python/orttraining_test_orttrainer_bert_toy_onnx.py b/orttraining/orttraining/test/python/orttraining_test_orttrainer_bert_toy_onnx.py index c5515f477d..45b87b32f7 100644 --- a/orttraining/orttraining/test/python/orttraining_test_orttrainer_bert_toy_onnx.py +++ b/orttraining/orttraining/test/python/orttraining_test_orttrainer_bert_toy_onnx.py @@ -112,7 +112,7 @@ def optimizer_parameters(model): no_decay_param_group = [] for initializer in model.graph.initializer: if any(key in initializer.name for key in no_decay_keys): - no_decay_param_group.append(initializer.name) # noqa: PERF401 + no_decay_param_group.append(initializer.name) params = [ { "params": no_decay_param_group, diff --git a/orttraining/tools/amdgpu/script/rocprof.py b/orttraining/tools/amdgpu/script/rocprof.py index a027ce4787..e5b107ba28 100644 --- a/orttraining/tools/amdgpu/script/rocprof.py +++ b/orttraining/tools/amdgpu/script/rocprof.py @@ -15,7 +15,7 @@ def get_gpu_lines(path): reader = csv.reader(f, delimiter=",") for row in reader: if row[2].find("TotalDurationNs") < 0: - lines.append(row) # noqa: PERF401 + lines.append(row) return lines diff --git a/orttraining/tools/scripts/gpt2_model_transform.py b/orttraining/tools/scripts/gpt2_model_transform.py index d707959138..06f03e0663 100644 --- a/orttraining/tools/scripts/gpt2_model_transform.py +++ b/orttraining/tools/scripts/gpt2_model_transform.py @@ -28,7 +28,7 @@ def find_input_node(model, arg): for node in model.graph.node: for output in node.output: if output == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -37,7 +37,7 @@ def find_output_node(model, arg): for node in model.graph.node: for input in node.input: if input == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -136,7 +136,7 @@ def process_concat(model): assert reshape_node.op_type == "Reshape" new_nodes[get_node_index(model, reshape_node)] = shape for n in fuse_nodes: - delete_nodes.append(get_node_index(model, n)) # noqa: PERF401 + delete_nodes.append(get_node_index(model, n)) # insert new shape to reshape index = 0 @@ -189,7 +189,7 @@ def fix_transpose(model): for n in model.graph.node: for input in n.input: if input == weight.name: - result.append(n) # noqa: PERF401 + result.append(n) if len(result) > 1: continue perm = node.attribute[0] @@ -280,7 +280,7 @@ def remove_input_ids_check_subgraph(model): remove_node_index = [] for n in removed_nodes: - remove_node_index.append(get_node_index(model, n)) # noqa: PERF401 + remove_node_index.append(get_node_index(model, n)) remove_node_index = list(set(remove_node_index)) remove_node_index.sort(reverse=True) diff --git a/orttraining/tools/scripts/layer_norm_transform.py b/orttraining/tools/scripts/layer_norm_transform.py index c3948b6378..b397d1d26a 100644 --- a/orttraining/tools/scripts/layer_norm_transform.py +++ b/orttraining/tools/scripts/layer_norm_transform.py @@ -141,7 +141,7 @@ def main(): all_nodes = [] for node in graph_proto.node: if node not in removed_nodes: - all_nodes.append(node) # noqa: PERF401 + all_nodes.append(node) for node in layer_norm_nodes: all_nodes.append(node) # noqa: PERF402 diff --git a/orttraining/tools/scripts/model_transform.py b/orttraining/tools/scripts/model_transform.py index 8ea2d5ab45..81e9f7b16b 100644 --- a/orttraining/tools/scripts/model_transform.py +++ b/orttraining/tools/scripts/model_transform.py @@ -26,7 +26,7 @@ def find_input_node(model, arg): for node in model.graph.node: for output in node.output: if output == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -35,7 +35,7 @@ def find_output_node(model, arg): for node in model.graph.node: for input in node.input: if input == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None @@ -94,7 +94,7 @@ def process_concat(model): if node.op_type == "Concat": input_nodes = [] for input in node.input: - input_nodes.append(find_input_node(model, input)) # noqa: PERF401 + input_nodes.append(find_input_node(model, input)) # figure out target shape shape = [] for input_node in input_nodes: @@ -116,7 +116,7 @@ def process_concat(model): assert reshape_node.op_type == "Reshape" new_nodes[get_node_index(model, reshape_node)] = shape for n in fuse_nodes: - delete_nodes.append(get_node_index(model, n)) # noqa: PERF401 + delete_nodes.append(get_node_index(model, n)) # insert new shape to reshape index = 0 for reshape_node_index in new_nodes: @@ -218,7 +218,7 @@ def fix_transpose(model): for n in model.graph.node: for input in n.input: if input == weight.name: - result.append(n) # noqa: PERF401 + result.append(n) if len(result) > 1: continue perm = node.attribute[0] @@ -242,7 +242,7 @@ def fix_transpose(model): old_ws = [] for t in transpose: if find_output_node(model, t[1].name) is None: - old_ws.append(find_weight_index(model, t[1].name)) # noqa: PERF401 + old_ws.append(find_weight_index(model, t[1].name)) old_ws.sort(reverse=True) for w_i in old_ws: del model.graph.initializer[w_i] diff --git a/orttraining/tools/scripts/opset12_model_transform.py b/orttraining/tools/scripts/opset12_model_transform.py index cda82b41b5..e8c2263a39 100644 --- a/orttraining/tools/scripts/opset12_model_transform.py +++ b/orttraining/tools/scripts/opset12_model_transform.py @@ -34,7 +34,7 @@ def find_input_node(model, arg): for node in model.graph.node: for output in node.output: if output == arg: - result.append(node) # noqa: PERF401 + result.append(node) return result[0] if len(result) == 1 else None diff --git a/orttraining/tools/scripts/performance_investigation.py b/orttraining/tools/scripts/performance_investigation.py index dfda008f6d..c8550a4d73 100644 --- a/orttraining/tools/scripts/performance_investigation.py +++ b/orttraining/tools/scripts/performance_investigation.py @@ -30,11 +30,11 @@ def process_file(onnx_file): if node.op_type == "ATen": for attr in node.attribute: if attr.name == "operator": - aten_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") # noqa: PERF401 + aten_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") if node.op_type == "PythonOp": for attr in node.attribute: if attr.name == "name": - python_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") # noqa: PERF401 + python_ops.append(f"{node.name}: {attr.s.decode('utf-8')}") # Look for stand-alone Dropout node in *_execution_model_.onnx graph. # Examine whether it should be fused with surrounding Add ops into BiasDropout node. diff --git a/orttraining/tools/scripts/pipeline_model_split.py b/orttraining/tools/scripts/pipeline_model_split.py index fb13463f1b..d1ae9dd22b 100644 --- a/orttraining/tools/scripts/pipeline_model_split.py +++ b/orttraining/tools/scripts/pipeline_model_split.py @@ -49,7 +49,7 @@ def split_graph(model, split_edge_groups): element_types.append(1) for info in model.graph.value_info: if info.name == id: - output_shapes.append(info.type) # noqa: PERF401 + output_shapes.append(info.type) send_input_signal_name = "send_input_signal" + str(cut_index) send_signal = model.graph.input.add() @@ -279,14 +279,14 @@ def generate_subgraph(model, start_nodes, identity_node_list): # remove added identity node before copy to subgraph identity_node_index = [] for n in identity_node_list: - identity_node_index.append(get_identity_index_for_deleting(main_graph.graph.node, n)) # noqa: PERF401 + identity_node_index.append(get_identity_index_for_deleting(main_graph.graph.node, n)) identity_node_index.sort(reverse=True) for i in reversed(range(len(main_graph.graph.node))): try: if i in identity_node_index: del main_graph.graph.node[i] - except Exception: # noqa: PERF203 + except Exception: print("error deleting identity node", i) all_visited_nodes = [] @@ -316,19 +316,19 @@ def generate_subgraph(model, start_nodes, identity_node_list): # gather visited nodes visited_nodes = [] for n in visited0: - visited_nodes.append(get_index(main_graph.graph.node, n)) # noqa: PERF401 + visited_nodes.append(get_index(main_graph.graph.node, n)) visited_nodes.sort(reverse=True) # gather visited inputs visited_inputs = [] for n in inputs0: - visited_inputs.append(get_index(main_graph.graph.input, n)) # noqa: PERF401 + visited_inputs.append(get_index(main_graph.graph.input, n)) visited_inputs.sort(reverse=True) # gather visited outputs visited_outputs = [] for n in outputs0: - visited_outputs.append(get_index(main_graph.graph.output, n)) # noqa: PERF401 + visited_outputs.append(get_index(main_graph.graph.output, n)) visited_outputs.sort(reverse=True) for i in reversed(range(len(main_graph.graph.node))): @@ -337,7 +337,7 @@ def generate_subgraph(model, start_nodes, identity_node_list): del subgraph.graph.node[i] else: del main_graph.graph.node[i] - except Exception: # noqa: PERF203 + except Exception: print("error deleting node", i) for i in reversed(range(len(main_graph.graph.input))): @@ -346,7 +346,7 @@ def generate_subgraph(model, start_nodes, identity_node_list): del subgraph.graph.input[i] else: del main_graph.graph.input[i] - except Exception: # noqa: PERF203 + except Exception: print("error deleting inputs", i) for i in reversed(range(len(main_graph.graph.output))): @@ -355,7 +355,7 @@ def generate_subgraph(model, start_nodes, identity_node_list): del subgraph.graph.output[i] else: del main_graph.graph.output[i] - except Exception: # noqa: PERF203 + except Exception: print("error deleting outputs ", i) print("model", str(model_count), " length ", len(subgraph.graph.node)) diff --git a/pyproject.toml b/pyproject.toml index dde001a176..ba07d3e3a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,8 @@ ignore = [ "N812", # Allow import torch.nn.functional as F "N999", # Module names "NPY002", # np.random.Generator may not always fit our use cases + "PERF203", # "try-except-in-loop" only affects Python <3.11, and the improvement is minor; can have false positives + "PERF401", # List comprehensions are not always readable "SIM102", # We don't perfer always combining if branches "SIM108", # We don't encourage ternary operators "SIM114", # Don't combine if branches for debugability diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index 3cb74ac2d2..acca4fb13c 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -69,7 +69,7 @@ def write_to_db(binary_size_data, args): branch_name = os.environ.get("BUILD_SOURCEBRANCHNAME", "main") rows = [] for row in binary_size_data: - rows.append( # noqa: PERF401 + rows.append( [ now_str, args.build_id, diff --git a/tools/doc/rename_folders.py b/tools/doc/rename_folders.py index 09ff7c4955..cc64775ae1 100644 --- a/tools/doc/rename_folders.py +++ b/tools/doc/rename_folders.py @@ -16,7 +16,7 @@ def rename_folder(root): for r, dirs, _files in os.walk(root): for name in dirs: if name.startswith("_"): - found.append((r, name)) # noqa: PERF401 + found.append((r, name)) renamed = [] for r, name in found: into = name.lstrip("_") diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index a4e00b9282..e6d4759769 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -87,9 +87,7 @@ def generate_file_list_for_ep(nuget_artifacts_dir, ep, files_list, include_pdbs, if child.name == "onnxruntime-android" or child.name == "onnxruntime-training-android": for child_file in child.iterdir(): if child_file.suffix in [".aar"]: - files_list.append( # noqa: PERF401 - '' - ) + files_list.append('') if child.name == "onnxruntime-ios-xcframework": files_list.append('') # noqa: ISC001 @@ -722,7 +720,7 @@ def generate_files(line_list, args): ngraph_list_path = os.path.join(openvino_path, "deployment_tools\\ngraph\\lib\\") for ngraph_element in os.listdir(ngraph_list_path): if ngraph_element.endswith("dll"): - files_list.append( # noqa: PERF401 + files_list.append( " GraphDef: for node in graph.node: _attr = [] for s in node.attribute: - _attr.append(" = ".join([str(f[1]) for f in s.ListFields()])) # noqa: PERF401 + _attr.append(" = ".join([str(f[1]) for f in s.ListFields()])) attr = ", ".join(_attr).encode(encoding="utf_8") shape_proto = None elem_type = 0 @@ -331,7 +331,7 @@ class ListUnpackTransformer(TransformerBase): new_output = f"{get_prefix(node.output[0])}{node.op_type}_{idx!s}_output" for output in node.output: if len(output) > 0: - new_nodes.append(helper.make_node("ListUnpack", [new_output], [output])) # noqa: PERF401 + new_nodes.append(helper.make_node("ListUnpack", [new_output], [output])) node.ClearField("output") node.output.extend([new_output]) if len(new_nodes) > 0: diff --git a/tools/python/util/convert_onnx_models_to_ort.py b/tools/python/util/convert_onnx_models_to_ort.py index d8329fca3c..18bba78661 100644 --- a/tools/python/util/convert_onnx_models_to_ort.py +++ b/tools/python/util/convert_onnx_models_to_ort.py @@ -165,7 +165,7 @@ def _convert( # new_size = os.path.getsize(ort_target_path) # print("Serialized {} to {}. Sizes: orig={} new={} diff={} new:old={:.4f}:1.0".format( # onnx_target_path, ort_target_path, orig_size, new_size, new_size - orig_size, new_size / orig_size)) - except Exception as e: # noqa: PERF203 + except Exception as e: print(f"Error converting {model}: {e}") if not allow_conversion_failures: raise diff --git a/tools/python/util/ort_format_model/operator_type_usage_processors.py b/tools/python/util/ort_format_model/operator_type_usage_processors.py index 08966f3d7b..5905000a14 100644 --- a/tools/python/util/ort_format_model/operator_type_usage_processors.py +++ b/tools/python/util/ort_format_model/operator_type_usage_processors.py @@ -205,7 +205,7 @@ class DefaultTypeUsageProcessor(TypeUsageProcessor): domain = _ort_constant_for_domain(self.domain) for i in sorted(self._input_types.keys()): if self._input_types[i]: - entries.append( # noqa: PERF401 + entries.append( "ORT_SPECIFY_OP_KERNEL_ARG_ALLOWED_TYPES({}, {}, Input, {}, {});".format( domain, self.optype, i, ", ".join(sorted(self._input_types[i])) ) @@ -213,7 +213,7 @@ class DefaultTypeUsageProcessor(TypeUsageProcessor): for o in sorted(self._output_types.keys()): if self._output_types[o]: - entries.append( # noqa: PERF401 + entries.append( "ORT_SPECIFY_OP_KERNEL_ARG_ALLOWED_TYPES({}, {}, Output, {}, {});".format( domain, self.optype, o, ", ".join(sorted(self._output_types[o])) )