mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-26 03:00:54 +00:00
a81faee41e
**Description**: This PR including following works: 1. provide stream and related synchronization abstractions in onnxruntime. 2. enhance onnxruntime's execution planner / executor / memory arena to support execute multiple streams in parallel. 3. deprecate the parallel executor for cpu. 4. deprecate the Fence mechanism. 5. update the cuda / tensorrt EP to support the stream mechanism, support running different request in different cuda stream. **Motivation and Context** - Why is this change required? currently, the execution plan is just a linear list of those primitives, ort will execute them step by step. For any given graph, ORT will serialize it to a fixed execution order. This sequential execution design simplifies most scenarios, but it has the following limitations: 1. it is difficult to enable inter-node parallelization, we have a half-baked parallel executor but it is very difficult to make it work with GPU. 2. The fence mechanism can work with single gpu stream + cpu thread case, but when extend to multiple stream, it is difficult to manage the cross GPU stream synchronizations. 3. our cuda EP rely on the BFCArena to make the memory management work with the GPU async kernels, but current BFCArena is not aware of the streams, so it doesn't behavior correctly when run with multiple streams. This PR enhance our existing execution plan and executor to support multiple stream execution. we use an unified algorithm to mange both single stream and multiple stream scenarios. This PR mainly focus on the infrastructure support for multiple stream execution, that is said, given a valid stream assignment, onnxruntime can execute it correctly. How to generate a good stream assignment for a given model will be in the future PR. Co-authored-by: Cheng Tang <chenta@microsoft.com@orttrainingdev9.d32nl1ml4oruzj4qz3bqlggovf.px.internal.cloudapp.net> Co-authored-by: Cheng Tang <chenta@microsoft.com> Co-authored-by: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Co-authored-by: Randy Shuai <rashuai@microsoft.com> Co-authored-by: cao lei <jslhcl@gmail.com> Co-authored-by: Lei Cao <leca@microsoft.com>
66 lines
1.8 KiB
C++
66 lines
1.8 KiB
C++
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// Licensed under the MIT License.
|
|
|
|
#pragma once
|
|
|
|
#include <functional>
|
|
#include <vector>
|
|
#include "core/common/common.h"
|
|
|
|
struct OrtDevice;
|
|
|
|
namespace onnxruntime {
|
|
#ifndef SHARED_PROVIDER
|
|
class Tensor;
|
|
#if !defined(DISABLE_SPARSE_TENSORS)
|
|
class SparseTensor;
|
|
#endif
|
|
#endif
|
|
class Stream;
|
|
|
|
namespace common {
|
|
class Status;
|
|
}
|
|
|
|
// Data transfer interface.
|
|
class IDataTransfer {
|
|
public:
|
|
virtual ~IDataTransfer() = default;
|
|
|
|
virtual bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const = 0;
|
|
|
|
virtual common::Status CopyTensor(const Tensor& src, Tensor& dst) const;
|
|
|
|
virtual common::Status CopyTensorAsync(const Tensor& /*src*/, Tensor& /*dst*/, Stream& /*stream*/) const {
|
|
ORT_NOT_IMPLEMENTED(__FUNCTION__, " is not implemented");
|
|
}
|
|
|
|
struct SrcDstPair {
|
|
std::reference_wrapper<const Tensor> src;
|
|
std::reference_wrapper<Tensor> dst;
|
|
Stream* src_stream; // producer stream of src
|
|
};
|
|
|
|
// batched copy. default implementation copies each entry sequentially, and returns on first failure.
|
|
virtual common::Status CopyTensors(const std::vector<SrcDstPair>& src_dst_pairs) const;
|
|
|
|
#if !defined(DISABLE_SPARSE_TENSORS)
|
|
struct SparseSrcDstPair {
|
|
std::reference_wrapper<const SparseTensor> src;
|
|
std::reference_wrapper<SparseTensor> dst;
|
|
int exec_queue_id;
|
|
};
|
|
|
|
virtual common::Status CopySparseTensors(const std::vector<SparseSrcDstPair>& src_dst_pairs) const;
|
|
#endif
|
|
};
|
|
|
|
class CPUDataTransfer : public IDataTransfer {
|
|
public:
|
|
CPUDataTransfer() = default;
|
|
// Dampen MSVC warning about not fully overriding CopyTensor
|
|
using IDataTransfer::CopyTensor;
|
|
bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const override;
|
|
common::Status CopyTensor(const Tensor& src, Tensor& dst) const override;
|
|
};
|
|
} // namespace onnxruntime
|