mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-19 21:32:23 +00:00
a81faee41e
**Description**: This PR including following works: 1. provide stream and related synchronization abstractions in onnxruntime. 2. enhance onnxruntime's execution planner / executor / memory arena to support execute multiple streams in parallel. 3. deprecate the parallel executor for cpu. 4. deprecate the Fence mechanism. 5. update the cuda / tensorrt EP to support the stream mechanism, support running different request in different cuda stream. **Motivation and Context** - Why is this change required? currently, the execution plan is just a linear list of those primitives, ort will execute them step by step. For any given graph, ORT will serialize it to a fixed execution order. This sequential execution design simplifies most scenarios, but it has the following limitations: 1. it is difficult to enable inter-node parallelization, we have a half-baked parallel executor but it is very difficult to make it work with GPU. 2. The fence mechanism can work with single gpu stream + cpu thread case, but when extend to multiple stream, it is difficult to manage the cross GPU stream synchronizations. 3. our cuda EP rely on the BFCArena to make the memory management work with the GPU async kernels, but current BFCArena is not aware of the streams, so it doesn't behavior correctly when run with multiple streams. This PR enhance our existing execution plan and executor to support multiple stream execution. we use an unified algorithm to mange both single stream and multiple stream scenarios. This PR mainly focus on the infrastructure support for multiple stream execution, that is said, given a valid stream assignment, onnxruntime can execute it correctly. How to generate a good stream assignment for a given model will be in the future PR. Co-authored-by: Cheng Tang <chenta@microsoft.com@orttrainingdev9.d32nl1ml4oruzj4qz3bqlggovf.px.internal.cloudapp.net> Co-authored-by: Cheng Tang <chenta@microsoft.com> Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Co-authored-by: Randy Shuai <rashuai@microsoft.com> Co-authored-by: cao lei <jslhcl@gmail.com> Co-authored-by: Lei Cao <leca@microsoft.com> |
||
|---|---|---|
| .. | ||
| allocation_planner.cc | ||
| allocation_planner.h | ||
| allocator.cc | ||
| allocator_stats.h | ||
| allocatormgr.cc | ||
| allocatormgr.h | ||
| arena_extend_strategy.h | ||
| bfc_arena.cc | ||
| bfc_arena.h | ||
| callback.cc | ||
| callback.h | ||
| compute_capability.h | ||
| config_options.cc | ||
| config_options.h | ||
| copy.cc | ||
| copy.h | ||
| customregistry.cc | ||
| data_transfer.cc | ||
| data_transfer.h | ||
| data_transfer_manager.cc | ||
| data_transfer_manager.h | ||
| data_transfer_utils.h | ||
| data_types.cc | ||
| debug_node_inputs_outputs_utils.cc | ||
| debug_node_inputs_outputs_utils.h | ||
| device_stream_collection.cc | ||
| device_stream_collection.h | ||
| element_type_lists.h | ||
| empty.cc | ||
| endian_utils.cc | ||
| endian_utils.h | ||
| error_code.cc | ||
| error_code_helper.h | ||
| ex_lib_loader.cc | ||
| ex_lib_loader.h | ||
| execution_frame.cc | ||
| execution_frame.h | ||
| execution_plan_base.h | ||
| execution_provider.cc | ||
| execution_providers.h | ||
| execution_steps.cc | ||
| execution_steps.h | ||
| fallback_cpu_capability.cc | ||
| fallback_cpu_capability.h | ||
| feeds_fetches_manager.cc | ||
| feeds_fetches_manager.h | ||
| func_kernel.cc | ||
| func_kernel.h | ||
| fuse_nodes_funcs.cc | ||
| fuse_nodes_funcs.h | ||
| graph_partitioner.cc | ||
| graph_partitioner.h | ||
| iexecutor.h | ||
| kernel_def_builder.cc | ||
| kernel_lookup.h | ||
| kernel_registry.cc | ||
| kernel_registry_manager.cc | ||
| kernel_registry_manager.h | ||
| kernel_type_str_resolver.cc | ||
| kernel_type_str_resolver.h | ||
| kernel_type_str_resolver_utils.cc | ||
| kernel_type_str_resolver_utils.h | ||
| math.h | ||
| mem_buffer.h | ||
| mem_pattern.h | ||
| mem_pattern_planner.h | ||
| memcpy.cc | ||
| memcpy.h | ||
| memory_info.cc | ||
| memory_info.h | ||
| mldata_type_utils.cc | ||
| mldata_type_utils.h | ||
| murmurhash3.cc | ||
| murmurhash3.h | ||
| node_index_info.cc | ||
| node_index_info.h | ||
| onnxruntime_map_type_info.cc | ||
| onnxruntime_map_type_info.h | ||
| onnxruntime_sequence_type_info.cc | ||
| onnxruntime_sequence_type_info.h | ||
| onnxruntime_typeinfo.cc | ||
| onnxruntime_typeinfo.h | ||
| op_kernel.cc | ||
| op_kernel_context_internal.h | ||
| op_kernel_info.cc | ||
| op_kernel_type_control_utils.h | ||
| op_node_proto_helper.cc | ||
| ort_stl_allocator.h | ||
| ort_value_name_idx_map.h | ||
| ort_value_pattern_planner.cc | ||
| ort_value_pattern_planner.h | ||
| ort_value_tensor_slicer.cc | ||
| ort_value_tensor_slicer.h | ||
| partial_graph_execution_state.cc | ||
| partial_graph_execution_state.h | ||
| prepacked_weights.cc | ||
| prepacked_weights.h | ||
| prepacked_weights_container.cc | ||
| prepacked_weights_container.h | ||
| print_tensor_utils.h | ||
| program_region.h | ||
| random_generator.cc | ||
| random_generator.h | ||
| random_seed.cc | ||
| random_seed.h | ||
| run_options.cc | ||
| sequential_execution_plan.h | ||
| sequential_executor.cc | ||
| sequential_executor.h | ||
| session_options.cc | ||
| session_options.h | ||
| session_state.cc | ||
| session_state.h | ||
| session_state_utils.cc | ||
| session_state_utils.h | ||
| simple_tensor_allocator.cc | ||
| simple_tensor_allocator.h | ||
| sparse_tensor.cc | ||
| sparse_utils.cc | ||
| sparse_utils.h | ||
| stream_execution_context.cc | ||
| stream_execution_context.h | ||
| tensor.cc | ||
| tensor_allocator.cc | ||
| tensor_allocator.h | ||
| tensor_allocator_with_mem_pattern.h | ||
| tensor_external_data_info.cc | ||
| tensor_external_data_info.h | ||
| tensor_shape.cc | ||
| tensor_type_and_shape.cc | ||
| tensor_type_and_shape.h | ||
| tensorprotoutils.cc | ||
| tensorprotoutils.h | ||
| TensorSeq.h | ||
| transpose_helper.cc | ||
| transpose_helper.h | ||
| tunable.h | ||
| utils.cc | ||
| utils.h | ||