mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-16 01:33:39 +00:00
a81faee41e
**Description**: This PR including following works: 1. provide stream and related synchronization abstractions in onnxruntime. 2. enhance onnxruntime's execution planner / executor / memory arena to support execute multiple streams in parallel. 3. deprecate the parallel executor for cpu. 4. deprecate the Fence mechanism. 5. update the cuda / tensorrt EP to support the stream mechanism, support running different request in different cuda stream. **Motivation and Context** - Why is this change required? currently, the execution plan is just a linear list of those primitives, ort will execute them step by step. For any given graph, ORT will serialize it to a fixed execution order. This sequential execution design simplifies most scenarios, but it has the following limitations: 1. it is difficult to enable inter-node parallelization, we have a half-baked parallel executor but it is very difficult to make it work with GPU. 2. The fence mechanism can work with single gpu stream + cpu thread case, but when extend to multiple stream, it is difficult to manage the cross GPU stream synchronizations. 3. our cuda EP rely on the BFCArena to make the memory management work with the GPU async kernels, but current BFCArena is not aware of the streams, so it doesn't behavior correctly when run with multiple streams. This PR enhance our existing execution plan and executor to support multiple stream execution. we use an unified algorithm to mange both single stream and multiple stream scenarios. This PR mainly focus on the infrastructure support for multiple stream execution, that is said, given a valid stream assignment, onnxruntime can execute it correctly. How to generate a good stream assignment for a given model will be in the future PR. Co-authored-by: Cheng Tang <chenta@microsoft.com@orttrainingdev9.d32nl1ml4oruzj4qz3bqlggovf.px.internal.cloudapp.net> Co-authored-by: Cheng Tang <chenta@microsoft.com> Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Co-authored-by: Randy Shuai <rashuai@microsoft.com> Co-authored-by: cao lei <jslhcl@gmail.com> Co-authored-by: Lei Cao <leca@microsoft.com>
84 lines
2.9 KiB
C++
84 lines
2.9 KiB
C++
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// Licensed under the MIT License.
|
|
|
|
#include "core/framework/data_transfer.h"
|
|
#ifndef SHARED_PROVIDER
|
|
#include "core/framework/tensor.h"
|
|
#include "core/framework/sparse_tensor.h"
|
|
#endif
|
|
#include "core/framework/ortdevice.h"
|
|
#ifdef ENABLE_STRIDED_TENSORS
|
|
#include "core/framework/copy.h"
|
|
#include "core/session/environment.h"
|
|
#include "core/common/logging/logging.h"
|
|
#include "core/common/logging/sinks/clog_sink.h"
|
|
#include "core/framework/element_type_lists.h"
|
|
#endif
|
|
|
|
namespace onnxruntime {
|
|
|
|
common::Status IDataTransfer::CopyTensor(const Tensor& /*src*/, Tensor& /*dst*/) const {
|
|
ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
|
|
}
|
|
|
|
common::Status IDataTransfer::CopyTensors(const std::vector<IDataTransfer::SrcDstPair>& src_dst_pairs) const {
|
|
for (const auto& pair : src_dst_pairs) {
|
|
if (pair.src_stream)
|
|
ORT_RETURN_IF_ERROR(CopyTensorAsync(pair.src, pair.dst, *pair.src_stream));
|
|
else
|
|
ORT_RETURN_IF_ERROR(CopyTensor(pair.src, pair.dst));
|
|
}
|
|
|
|
return Status::OK();
|
|
}
|
|
|
|
#if !defined(DISABLE_SPARSE_TENSORS)
|
|
common::Status IDataTransfer::CopySparseTensors(const std::vector<SparseSrcDstPair>& src_dst_pairs) const {
|
|
for (const auto& pair : src_dst_pairs) {
|
|
ORT_RETURN_IF_ERROR(pair.src.get().Copy(*this, pair.dst));
|
|
}
|
|
return Status::OK();
|
|
}
|
|
#endif
|
|
|
|
bool CPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const {
|
|
return src_device.Type() == OrtDevice::CPU && dst_device.Type() == OrtDevice::CPU;
|
|
}
|
|
|
|
common::Status CPUDataTransfer::CopyTensor(const Tensor& src, Tensor& dst) const {
|
|
const void* src_data = src.DataRaw();
|
|
void* dst_data = dst.MutableDataRaw();
|
|
if (src_data == dst_data) {
|
|
// no need copying as both pointers are referring to same piece of memory.
|
|
return Status::OK();
|
|
}
|
|
|
|
#ifdef ENABLE_STRIDED_TENSORS
|
|
if (!src.IsContiguous() || !dst.IsContiguous()) {
|
|
auto dst_stride_vec = dst.Strides();
|
|
auto src_stride_vec = src.Strides();
|
|
onnxruntime::TensorShapeVector dst_stride{dst_stride_vec.begin(), dst_stride_vec.end()};
|
|
onnxruntime::TensorShapeVector src_stride{src_stride_vec.begin(), src_stride_vec.end()};
|
|
return DispatchStridedCopy<element_type_lists::All>(nullptr,
|
|
dst, 0, dst_stride,
|
|
src.Shape(),
|
|
src, 0, src_stride);
|
|
} else {
|
|
#endif
|
|
// Copying only happens between two same size tensors.
|
|
ORT_ENFORCE(src.SizeInBytes() == dst.SizeInBytes());
|
|
if (!src.IsDataTypeString()) {
|
|
memcpy(dst_data, src_data, src.SizeInBytes());
|
|
} else {
|
|
const auto* src_strings = src.Data<std::string>();
|
|
auto* dst_strings = dst.MutableData<std::string>();
|
|
std::copy(src_strings, src_strings + src.Shape().Size(), dst_strings);
|
|
}
|
|
|
|
return Status::OK();
|
|
#ifdef ENABLE_STRIDED_TENSORS
|
|
}
|
|
#endif
|
|
}
|
|
|
|
}; // namespace onnxruntime
|