saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-08 17:17:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add tensor v2 support (#20530)

Browse source 
### Description
Add tensor v2 support to unblock the inference with context binary
generated from QNN v2.21
This commit is contained in:
Hector Li 2024-05-02 13:49:04 -07:00 committed by GitHub
parent 589aeb7036
commit e6228575e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 206 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

250
onnxruntime/core/providers/qnn/builder/qnn_def.cc
View file

@ -23,50 +23,99 @@ size_t memscpy(void* dst, size_t dst_size, const void* src, size_t copy_size) {
void SetQnnTensorType(Qnn_Tensor_t& qnn_tensor, Qnn_TensorType_t tensor_type) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.type = tensor_type;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.type = tensor_type;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorName(Qnn_Tensor_t& qnn_tensor, const char* name) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.name = name;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.name = name;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorDataFormat(Qnn_Tensor_t& qnn_tensor, Qnn_TensorDataFormat_t data_format) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.dataFormat = data_format;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.dataFormat = data_format;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorDataType(Qnn_Tensor_t& qnn_tensor, Qnn_DataType_t data_type) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.dataType = data_type;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.dataType = data_type;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorDim(Qnn_Tensor_t& qnn_tensor, const std::vector<uint32_t>& dimensions) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.rank = static_cast<uint32_t>(dimensions.size());
qnn_tensor.v1.dimensions = const_cast<uint32_t*>(dimensions.data());
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.rank = static_cast<uint32_t>(dimensions.size());
qnn_tensor.v2.dimensions = const_cast<uint32_t*>(dimensions.data());
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorMemType(Qnn_Tensor_t& qnn_tensor, Qnn_TensorMemType_t mem_type) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.memType = mem_type;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.memType = mem_type;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorClientBuf(Qnn_Tensor_t& qnn_tensor, const std::vector<uint8_t>& client_buf) {
@ -74,9 +123,19 @@ void SetQnnTensorClientBuf(Qnn_Tensor_t& qnn_tensor, const std::vector<uint8_t>&
auto size = client_buf.size() * sizeof(uint8_t);
qnn_tensor.v1.clientBuf.data = const_cast<void*>(static_cast<const void*>(client_buf.data()));
qnn_tensor.v1.clientBuf.dataSize = static_cast<uint32_t>(size);
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
auto size = client_buf.size() * sizeof(uint8_t);
qnn_tensor.v2.clientBuf.data = const_cast<void*>(static_cast<const void*>(client_buf.data()));
qnn_tensor.v2.clientBuf.dataSize = static_cast<uint32_t>(size);
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorClientBuf(Qnn_Tensor_t& qnn_tensor, const std::vector<uint32_t>& client_buf) {
@ -84,122 +143,225 @@ void SetQnnTensorClientBuf(Qnn_Tensor_t& qnn_tensor, const std::vector<uint32_t>
auto size = client_buf.size() * sizeof(uint32_t);
qnn_tensor.v1.clientBuf.data = const_cast<void*>(static_cast<const void*>(client_buf.data()));
qnn_tensor.v1.clientBuf.dataSize = static_cast<uint32_t>(size);
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
auto size = client_buf.size() * sizeof(uint32_t);
qnn_tensor.v2.clientBuf.data = const_cast<void*>(static_cast<const void*>(client_buf.data()));
qnn_tensor.v2.clientBuf.dataSize = static_cast<uint32_t>(size);
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorClientBuf(Qnn_Tensor_t& qnn_tensor, void* buf_data, uint32_t buf_size) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.clientBuf.data = buf_data;
qnn_tensor.v1.clientBuf.dataSize = buf_size;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.clientBuf.data = buf_data;
qnn_tensor.v2.clientBuf.dataSize = buf_size;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorClientBufSize(Qnn_Tensor_t& qnn_tensor, uint32_t client_buf_size) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.clientBuf.dataSize = client_buf_size;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.clientBuf.dataSize = client_buf_size;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorClientBufData(Qnn_Tensor_t& qnn_tensor, void* client_buf_data) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.clientBuf.data = client_buf_data;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.clientBuf.data = client_buf_data;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
void SetQnnTensorQParams(Qnn_Tensor_t& qnn_tensor, const Qnn_QuantizeParams_t& quantize_params) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
qnn_tensor.v1.quantizeParams = quantize_params;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
return;
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
qnn_tensor.v2.quantizeParams = quantize_params;
return;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
uint32_t GetQnnTensorID(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.id;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.id;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
Qnn_TensorType_t GetQnnTensorType(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.type;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.type;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
const char* GetQnnTensorName(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.name;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.name;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
Qnn_TensorDataFormat_t GetQnnTensorDataFormat(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.dataFormat;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.dataFormat;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
Qnn_DataType_t GetQnnTensorDataType(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.dataType;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.dataType;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
Qnn_TensorMemType_t GetQnnTensorMemType(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.memType;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.memType;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
uint32_t GetQnnTensorRank(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.rank;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.rank;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
uint32_t* GetQnnTensorDims(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.dimensions;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.dimensions;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
const Qnn_ClientBuffer_t& GetQnnTensorClientBuf(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.clientBuf;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.clientBuf;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
const Qnn_QuantizeParams_t& GetQnnTensorQParams(const Qnn_Tensor_t& qnn_tensor) {
if (QNN_TENSOR_VERSION_1 == qnn_tensor.version) {
return qnn_tensor.v1.quantizeParams;
} else {
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
#ifdef QNN_TENSOR_V2_INIT
if (QNN_TENSOR_VERSION_2 == qnn_tensor.version) {
return qnn_tensor.v2.quantizeParams;
}
#endif // QNN_TENSOR_V2_INIT
ORT_THROW("QNN tensor version not supported, QNN tensor version: ", qnn_tensor.version);
}
Status CompareQnnQuantParams(const Qnn_QuantizeParams_t& qparam0, const Qnn_QuantizeParams_t& qparam1,