saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-25 19:48:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

QDQ implementation (#7033)

Browse source 
* Add QDQ basic implementation
This commit is contained in:
Yufeng Li 2021-03-25 09:17:23 -07:00 committed by GitHub
parent 865c67611c
commit 8e54b76e2d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 1379 additions and 1054 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmake/onnxruntime_optimizer.cmake
View file

@ -13,6 +13,8 @@ else()
"${ONNXRUNTIME_INCLUDE_DIR}/core/optimizer/*.h"
"${ONNXRUNTIME_ROOT}/core/optimizer/*.h"
"${ONNXRUNTIME_ROOT}/core/optimizer/*.cc"
"${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.h"
"${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.cc"
)
endif()

21
onnxruntime/core/common/type_utils.h Normal file
View file

@ -0,0 +1,21 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <cstdint>
#include <type_traits>
namespace onnxruntime {
namespace utils {
template <typename T>
struct IsByteType : std::false_type {};
template <>
struct IsByteType<uint8_t> : std::true_type {};
template <>
struct IsByteType<int8_t> : std::true_type {};
} // namespace utils
} // namespace onnxruntime

611
onnxruntime/core/framework/cblas.h
View file

@ -1,611 +0,0 @@
/**
* Copyright (c) 2016-present, Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* Modifications Copyright (c) Microsoft. */
// This is the exact cblas.h header file, placed here purely in order to get
// the enums.
#ifndef CBLAS_H
#ifndef CBLAS_ENUM_DEFINED_H
#define CBLAS_ENUM_DEFINED_H
enum CBLAS_ORDER { CblasRowMajor = 101,
CblasColMajor = 102 };
enum CBLAS_TRANSPOSE {
CblasNoTrans = 111,
CblasTrans = 112,
CblasConjTrans = 113
};
enum CBLAS_UPLO { CblasUpper = 121,
CblasLower = 122 };
enum CBLAS_DIAG { CblasNonUnit = 131,
CblasUnit = 132 };
enum CBLAS_SIDE { CblasLeft = 141,
CblasRight = 142 };
#endif
#ifndef CBLAS_ENUM_ONLY
#define CBLAS_H
#define CBLAS_INDEX int
int cblas_errprn(int ierr, int info, char* form, ...);
void cblas_xerbla(int p, const char* rout, const char* form, ...);
/*
* ===========================================================================
* Prototypes for level 1 BLAS functions (complex are recast as routines)
* ===========================================================================
*/
float cblas_sdsdot(int N, float alpha, const float* X,
int incX, const float* Y, int incY);
double cblas_dsdot(int N, const float* X, int incX, const float* Y,
int incY);
float cblas_sdot(int N, const float* X, int incX,
const float* Y, int incY);
double cblas_ddot(int N, const double* X, int incX,
const double* Y, int incY);
/*
* Functions having prefixes Z and C only
*/
void cblas_cdotu_sub(int N, const void* X, int incX,
const void* Y, int incY, void* dotu);
void cblas_cdotc_sub(int N, const void* X, int incX,
const void* Y, int incY, void* dotc);
void cblas_zdotu_sub(int N, const void* X, int incX,
const void* Y, int incY, void* dotu);
void cblas_zdotc_sub(int N, const void* X, int incX,
const void* Y, int incY, void* dotc);
/*
* Functions having prefixes S D SC DZ
*/
float cblas_snrm2(int N, const float* X, int incX);
float cblas_sasum(int N, const float* X, int incX);
double cblas_dnrm2(int N, const double* X, int incX);
double cblas_dasum(int N, const double* X, int incX);
float cblas_scnrm2(int N, const void* X, int incX);
float cblas_scasum(int N, const void* X, int incX);
double cblas_dznrm2(int N, const void* X, int incX);
double cblas_dzasum(int N, const void* X, int incX);
/*
* Functions having standard 4 prefixes (S D C Z)
*/
CBLAS_INDEX cblas_isamax(int N, const float* X, int incX);
CBLAS_INDEX cblas_idamax(int N, const double* X, int incX);
CBLAS_INDEX cblas_icamax(int N, const void* X, int incX);
CBLAS_INDEX cblas_izamax(int N, const void* X, int incX);
/*
* ===========================================================================
* Prototypes for level 1 BLAS routines
* ===========================================================================
*/
/*
* Routines with standard 4 prefixes (s, d, c, z)
*/
void cblas_sswap(int N, float* X, int incX,
float* Y, int incY);
void cblas_scopy(int N, const float* X, int incX,
float* Y, int incY);
void cblas_saxpy(int N, float alpha, const float* X,
int incX, float* Y, int incY);
void catlas_saxpby(int N, float alpha, const float* X,
int incX, float beta, float* Y, int incY);
void catlas_sset(int N, float alpha, float* X, int incX);
void cblas_dswap(int N, double* X, int incX,
double* Y, int incY);
void cblas_dcopy(int N, const double* X, int incX,
double* Y, int incY);
void cblas_daxpy(int N, double alpha, const double* X,
int incX, double* Y, int incY);
void catlas_daxpby(int N, double alpha, const double* X,
int incX, double beta, double* Y, int incY);
void catlas_dset(int N, double alpha, double* X, int incX);
void cblas_cswap(int N, void* X, int incX,
void* Y, int incY);
void cblas_ccopy(int N, const void* X, int incX,
void* Y, int incY);
void cblas_caxpy(int N, const void* alpha, const void* X,
int incX, void* Y, int incY);
void catlas_caxpby(int N, const void* alpha, const void* X,
int incX, const void* beta, void* Y, int incY);
void catlas_cset(int N, const void* alpha, void* X, int incX);
void cblas_zswap(int N, void* X, int incX,
void* Y, int incY);
void cblas_zcopy(int N, const void* X, int incX,
void* Y, int incY);
void cblas_zaxpy(int N, const void* alpha, const void* X,
int incX, void* Y, int incY);
void catlas_zaxpby(int N, const void* alpha, const void* X,
int incX, const void* beta, void* Y, int incY);
void catlas_zset(int N, const void* alpha, void* X, int incX);
/*
* Routines with S and D prefix only
*/
void cblas_srotg(float* a, float* b, float* c, float* s);
void cblas_srotmg(float* d1, float* d2, float* b1, float b2, float* P);
void cblas_srot(int N, float* X, int incX,
float* Y, int incY, float c, float s);
void cblas_srotm(int N, float* X, int incX,
float* Y, int incY, const float* P);
void cblas_drotg(double* a, double* b, double* c, double* s);
void cblas_drotmg(double* d1, double* d2, double* b1, double b2, double* P);
void cblas_drot(int N, double* X, int incX,
double* Y, int incY, double c, double s);
void cblas_drotm(int N, double* X, int incX,
double* Y, int incY, const double* P);
/*
* Routines with S D C Z CS and ZD prefixes
*/
void cblas_sscal(int N, float alpha, float* X, int incX);
void cblas_dscal(int N, double alpha, double* X, int incX);
void cblas_cscal(int N, const void* alpha, void* X, int incX);
void cblas_zscal(int N, const void* alpha, void* X, int incX);
void cblas_csscal(int N, float alpha, void* X, int incX);
void cblas_zdscal(int N, double alpha, void* X, int incX);
/*
* Extra reference routines provided by ATLAS, but not mandated by the standard
*/
void cblas_crotg(void* a, void* b, void* c, void* s);
void cblas_zrotg(void* a, void* b, void* c, void* s);
void cblas_csrot(int N, void* X, int incX, void* Y, int incY,
float c, float s);
void cblas_zdrot(int N, void* X, int incX, void* Y, int incY,
double c, double s);
/*
* ===========================================================================
* Prototypes for level 2 BLAS
* ===========================================================================
*/
/*
* Routines with standard 4 prefixes (S, D, C, Z)
*/
void cblas_sgemv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
float alpha, const float* A, int lda,
const float* X, int incX, float beta,
float* Y, int incY);
void cblas_sgbmv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
int KL, int KU, float alpha,
const float* A, int lda, const float* X,
int incX, float beta, float* Y, int incY);
void cblas_strmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const float* A, int lda,
float* X, int incX);
void cblas_stbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const float* A, int lda,
float* X, int incX);
void cblas_stpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const float* Ap, float* X, int incX);
void cblas_strsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const float* A, int lda, float* X,
int incX);
void cblas_stbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const float* A, int lda,
float* X, int incX);
void cblas_stpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const float* Ap, float* X, int incX);
void cblas_dgemv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
double alpha, const double* A, int lda,
const double* X, int incX, double beta,
double* Y, int incY);
void cblas_dgbmv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
int KL, int KU, double alpha,
const double* A, int lda, const double* X,
int incX, double beta, double* Y, int incY);
void cblas_dtrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const double* A, int lda,
double* X, int incX);
void cblas_dtbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const double* A, int lda,
double* X, int incX);
void cblas_dtpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const double* Ap, double* X, int incX);
void cblas_dtrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const double* A, int lda, double* X,
int incX);
void cblas_dtbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const double* A, int lda,
double* X, int incX);
void cblas_dtpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const double* Ap, double* X, int incX);
void cblas_cgemv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
const void* alpha, const void* A, int lda,
const void* X, int incX, const void* beta,
void* Y, int incY);
void cblas_cgbmv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
int KL, int KU, const void* alpha,
const void* A, int lda, const void* X,
int incX, const void* beta, void* Y, int incY);
void cblas_ctrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* A, int lda,
void* X, int incX);
void cblas_ctbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const void* A, int lda,
void* X, int incX);
void cblas_ctpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* Ap, void* X, int incX);
void cblas_ctrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* A, int lda, void* X,
int incX);
void cblas_ctbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const void* A, int lda,
void* X, int incX);
void cblas_ctpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* Ap, void* X, int incX);
void cblas_zgemv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
const void* alpha, const void* A, int lda,
const void* X, int incX, const void* beta,
void* Y, int incY);
void cblas_zgbmv(enum CBLAS_ORDER Order,
enum CBLAS_TRANSPOSE TransA, int M, int N,
int KL, int KU, const void* alpha,
const void* A, int lda, const void* X,
int incX, const void* beta, void* Y, int incY);
void cblas_ztrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* A, int lda,
void* X, int incX);
void cblas_ztbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const void* A, int lda,
void* X, int incX);
void cblas_ztpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* Ap, void* X, int incX);
void cblas_ztrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* A, int lda, void* X,
int incX);
void cblas_ztbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, int K, const void* A, int lda,
void* X, int incX);
void cblas_ztpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag,
int N, const void* Ap, void* X, int incX);
/*
* Routines with S and D prefixes only
*/
void cblas_ssymv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* A,
int lda, const float* X, int incX,
float beta, float* Y, int incY);
void cblas_ssbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, int K, float alpha, const float* A,
int lda, const float* X, int incX,
float beta, float* Y, int incY);
void cblas_sspmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* Ap,
const float* X, int incX,
float beta, float* Y, int incY);
void cblas_sger(enum CBLAS_ORDER Order, int M, int N,
float alpha, const float* X, int incX,
const float* Y, int incY, float* A, int lda);
void cblas_ssyr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* X,
int incX, float* A, int lda);
void cblas_sspr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* X,
int incX, float* Ap);
void cblas_ssyr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* X,
int incX, const float* Y, int incY, float* A,
int lda);
void cblas_sspr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const float* X,
int incX, const float* Y, int incY, float* A);
void cblas_dsymv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* A,
int lda, const double* X, int incX,
double beta, double* Y, int incY);
void cblas_dsbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, int K, double alpha, const double* A,
int lda, const double* X, int incX,
double beta, double* Y, int incY);
void cblas_dspmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* Ap,
const double* X, int incX,
double beta, double* Y, int incY);
void cblas_dger(enum CBLAS_ORDER Order, int M, int N,
double alpha, const double* X, int incX,
const double* Y, int incY, double* A, int lda);
void cblas_dsyr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* X,
int incX, double* A, int lda);
void cblas_dspr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* X,
int incX, double* Ap);
void cblas_dsyr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* X,
int incX, const double* Y, int incY, double* A,
int lda);
void cblas_dspr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const double* X,
int incX, const double* Y, int incY, double* A);
/*
* Routines with C and Z prefixes only
*/
void cblas_chemv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, const void* alpha, const void* A,
int lda, const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_chbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, int K, const void* alpha, const void* A,
int lda, const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_chpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, const void* alpha, const void* Ap,
const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_cgeru(enum CBLAS_ORDER Order, int M, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_cgerc(enum CBLAS_ORDER Order, int M, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_cher(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const void* X, int incX,
void* A, int lda);
void cblas_chpr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, float alpha, const void* X,
int incX, void* A);
void cblas_cher2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_chpr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* Ap);
void cblas_zhemv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, const void* alpha, const void* A,
int lda, const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_zhbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, int K, const void* alpha, const void* A,
int lda, const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_zhpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, const void* alpha, const void* Ap,
const void* X, int incX,
const void* beta, void* Y, int incY);
void cblas_zgeru(enum CBLAS_ORDER Order, int M, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_zgerc(enum CBLAS_ORDER Order, int M, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_zher(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const void* X, int incX,
void* A, int lda);
void cblas_zhpr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
int N, double alpha, const void* X,
int incX, void* A);
void cblas_zher2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* A, int lda);
void cblas_zhpr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N,
const void* alpha, const void* X, int incX,
const void* Y, int incY, void* Ap);
/*
* ===========================================================================
* Prototypes for level 3 BLAS
* ===========================================================================
*/
/*
* Routines with standard 4 prefixes (S, D, C, Z)
*/
void cblas_sgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_TRANSPOSE TransB, int M, int N,
int K, float alpha, const float* A,
int lda, const float* B, int ldb,
float beta, float* C, int ldc);
void cblas_ssymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
float alpha, const float* A, int lda,
const float* B, int ldb, float beta,
float* C, int ldc);
void cblas_ssyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
float alpha, const float* A, int lda,
float beta, float* C, int ldc);
void cblas_ssyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
float alpha, const float* A, int lda,
const float* B, int ldb, float beta,
float* C, int ldc);
void cblas_strmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
float alpha, const float* A, int lda,
float* B, int ldb);
void cblas_strsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
float alpha, const float* A, int lda,
float* B, int ldb);
void cblas_dgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_TRANSPOSE TransB, int M, int N,
int K, double alpha, const double* A,
int lda, const double* B, int ldb,
double beta, double* C, int ldc);
void cblas_dsymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
double alpha, const double* A, int lda,
const double* B, int ldb, double beta,
double* C, int ldc);
void cblas_dsyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
double alpha, const double* A, int lda,
double beta, double* C, int ldc);
void cblas_dsyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
double alpha, const double* A, int lda,
const double* B, int ldb, double beta,
double* C, int ldc);
void cblas_dtrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
double alpha, const double* A, int lda,
double* B, int ldb);
void cblas_dtrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
double alpha, const double* A, int lda,
double* B, int ldb);
void cblas_cgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_TRANSPOSE TransB, int M, int N,
int K, const void* alpha, const void* A,
int lda, const void* B, int ldb,
const void* beta, void* C, int ldc);
void cblas_csymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_csyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* beta, void* C, int ldc);
void cblas_csyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_ctrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
const void* alpha, const void* A, int lda,
void* B, int ldb);
void cblas_ctrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
const void* alpha, const void* A, int lda,
void* B, int ldb);
void cblas_zgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_TRANSPOSE TransB, int M, int N,
int K, const void* alpha, const void* A,
int lda, const void* B, int ldb,
const void* beta, void* C, int ldc);
void cblas_zsymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_zsyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* beta, void* C, int ldc);
void cblas_zsyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_ztrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
const void* alpha, const void* A, int lda,
void* B, int ldb);
void cblas_ztrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA,
enum CBLAS_DIAG Diag, int M, int N,
const void* alpha, const void* A, int lda,
void* B, int ldb);
/*
* Routines with prefixes C and Z only
*/
void cblas_chemm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_cherk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
float alpha, const void* A, int lda,
float beta, void* C, int ldc);
void cblas_cher2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* B, int ldb, float beta,
void* C, int ldc);
void cblas_zhemm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side,
enum CBLAS_UPLO Uplo, int M, int N,
const void* alpha, const void* A, int lda,
const void* B, int ldb, const void* beta,
void* C, int ldc);
void cblas_zherk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
double alpha, const void* A, int lda,
double beta, void* C, int ldc);
void cblas_zher2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo,
enum CBLAS_TRANSPOSE Trans, int N, int K,
const void* alpha, const void* A, int lda,
const void* B, int ldb, double beta,
void* C, int ldc);
#endif /* end #ifdef CBLAS_ENUM_ONLY */
#endif

36
onnxruntime/core/graph/graph_utils.cc
View file

@ -565,6 +565,26 @@ const Node* FirstChildByType(const Node& node, const std::string& child_type) {
return nullptr;
}
std::vector<const Node*> FindChildrenByType(const Node& node, const std::string& child_type) {
// find children and sort them by source argument index:
// Create a 2D vector to hold the result.
// 1st dimension index is output index,
// and the 2nd dimension stores the edges from the output.
std::vector<std::vector<const Node*>> children(node.OutputDefs().size(), std::vector<const Node*>());
for (auto it = node.OutputEdgesBegin(); it != node.OutputEdgesEnd(); it++) {
if (it->GetNode().OpType().compare(child_type) == 0) {
children[it->GetSrcArgIndex()].push_back(&(it->GetNode()));
}
}
// aggregate children
std::vector<const Node*> agg_res;
for (size_t output_idx = 0; output_idx < children.size(); output_idx++) {
agg_res.insert(agg_res.end(), children[output_idx].begin(), children[output_idx].end());
}
return agg_res;
}
const Node* FirstParentByType(const Node& node, const std::string& parent_type) {
for (auto it = node.InputNodesBegin(); it != node.InputNodesEnd(); ++it) {
if ((*it).OpType().compare(parent_type) == 0) {
@ -574,6 +594,22 @@ const Node* FirstParentByType(const Node& node, const std::string& parent_type)
return nullptr;
}
std::vector<const Node*> FindParentsByType(const Node& node, const std::string& parent_type) {
// find parents and sort them by destination argument index
// as there is at most one input edge for each input argument,
// there is no need of extra work like FindChildrenByType
std::vector<const Node*> parents(node.InputDefs().size(), nullptr);
for (auto it = node.InputEdgesBegin(); it != node.InputEdgesEnd(); it++) {
if (it->GetNode().OpType().compare(parent_type) == 0) {
parents[it->GetDstArgIndex()] = &(it->GetNode());
}
}
// remove unmatched nodes
parents.erase(std::remove(parents.begin(), parents.end(), nullptr), parents.end());
return parents;
}
NodeArg& AddInitializer(Graph& graph, const ONNX_NAMESPACE::TensorProto& new_initializer) {
// sanity check as AddInitializedTensor silently ignores attempts to add a duplicate initializer
const ONNX_NAMESPACE::TensorProto* existing = nullptr;

11
onnxruntime/core/graph/graph_utils.h
View file

@ -100,9 +100,20 @@ bool GetRepeatedNodeAttributeValues(const Node& node,
/** Find the first child of the specified op type. */
const Node* FirstChildByType(const Node& node, const std::string& child_type);
/** Find node children by op types.
@returns The matched children are sorted by source argument index of their corresponding edge.
**/
std::vector<const Node*> FindChildrenByType(const Node& node, const std::string& child_type);
/** Find the first parent of the specified op type. */
const Node* FirstParentByType(const Node& node, const std::string& parent_type);
/** Find node parents by op types.
@returns The matched parents are sorted by destination argument index of their corresponding edge.
**/
std::vector<const Node*> FindParentsByType(const Node& node, const std::string& parent_type);
/** Tests if we can remove a node and merge its input edge (if any) with its output edges.
Conditions:
Input rules:

5
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -37,6 +37,7 @@
#include "core/optimizer/skip_layer_norm_fusion.h"
#include "core/optimizer/slice_elimination.h"
#include "core/optimizer/unsqueeze_elimination.h"
#include "core/optimizer/qdq_transformer/qdq_transformer.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/optimizer/matmul_transpose_fusion.h"
#include "core/optimizer/bias_dropout_fusion.h"
@ -143,6 +144,10 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
transformers.emplace_back(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(cpu_execution_providers));
transformers.emplace_back(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(cpu_execution_providers));
if (enable_quant_qdq) {
transformers.emplace_back(onnxruntime::make_unique<QDQTransformer>());
}
std::unordered_set<std::string> cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider};
std::unordered_set<std::string> cpu_cuda_rocm_acl_armnn_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider};

46
onnxruntime/core/optimizer/qdq_transformer/qdq_binary_op.cc Normal file
View file

@ -0,0 +1,46 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <vector>
#include "core/graph/graph.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
#include "core/optimizer/qdq_transformer/registry.h"
namespace onnxruntime {
class QDQBinaryOpTransformer : public QDQOperatorTransformer {
public:
QDQBinaryOpTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {}
bool Transform(const std::vector<const Node*>& parents, const std::vector<const Node*>& children) override {
if (children.size() != 1 || parents.size() != 2) {
return false;
}
FillQDQOptionalZeroPoint(parents);
FillQDQOptionalZeroPoint(children);
std::vector<NodeArg*> input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs());
Node* b = graph_.GetNode(parents[1]->Index());
input_defs.insert(input_defs.end(), b->MutableInputDefs().begin(), b->MutableInputDefs().end());
Node* q = graph_.GetNode(children[0]->Index());
input_defs.push_back(q->MutableInputDefs()[1]);
input_defs.push_back(q->MutableInputDefs()[2]);
Node& qlinear_conv_node = graph_.AddNode(node_.Name(),
"QLinear" + node_.OpType(),
node_.Description(),
input_defs,
q->MutableOutputDefs(),
&node_.GetAttributes(),
kMSDomain);
qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider);
return true;
}
};
DEFINE_QDQ_CREATOR(Add, QDQBinaryOpTransformer)
DEFINE_QDQ_CREATOR(Mul, QDQBinaryOpTransformer)
} // namespace onnxruntime

49
onnxruntime/core/optimizer/qdq_transformer/qdq_conv.cc Normal file
View file

@ -0,0 +1,49 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <vector>
#include "core/graph/graph.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
#include "core/optimizer/qdq_transformer/registry.h"
namespace onnxruntime {
class QDQConvTransformer : public QDQOperatorTransformer {
public:
QDQConvTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {}
bool Transform(const std::vector<const Node*>& parents, const std::vector<const Node*>& children) override {
if (parents.size() < 2 || children.size() != 1) {
return false;
}
FillQDQOptionalZeroPoint(parents);
FillQDQOptionalZeroPoint(children);
std::vector<NodeArg*> input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs());
Node* weight = graph_.GetNode(parents[1]->Index());
input_defs.insert(input_defs.end(), weight->MutableInputDefs().begin(), weight->MutableInputDefs().end());
Node* q = graph_.GetNode(children[0]->Index());
input_defs.push_back(q->MutableInputDefs()[1]);
input_defs.push_back(q->MutableInputDefs()[2]);
if (parents.size() == 3) {
input_defs.push_back(graph_.GetNode(parents[2]->Index())->MutableInputDefs()[0]);
}
Node& qlinear_conv_node = graph_.AddNode(node_.Name(),
"QLinearConv",
node_.Description(),
input_defs,
q->MutableOutputDefs(),
&node_.GetAttributes(),
kOnnxDomain);
qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider);
return true;
}
};
DEFINE_QDQ_CREATOR(Conv, QDQConvTransformer)
} // namespace onnxruntime

44
onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc Normal file
View file

@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <vector>
#include "core/graph/graph.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
#include "core/optimizer/qdq_transformer/registry.h"
namespace onnxruntime {
class QDQMatMulTransformer : public QDQOperatorTransformer {
public:
QDQMatMulTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {}
bool Transform(const std::vector<const Node*>& parents, const std::vector<const Node*>& children) override {
if (children.size() != 1) {
return false;
}
FillQDQOptionalZeroPoint(parents);
FillQDQOptionalZeroPoint(children);
std::vector<NodeArg*> input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs());
Node* b = graph_.GetNode(parents[1]->Index());
input_defs.insert(input_defs.end(), b->MutableInputDefs().begin(), b->MutableInputDefs().end());
Node* q = graph_.GetNode(children[0]->Index());
input_defs.push_back(q->MutableInputDefs()[1]);
input_defs.push_back(q->MutableInputDefs()[2]);
Node& qlinear_conv_node = graph_.AddNode(node_.Name(),
"QLinearMatMul",
node_.Description(),
input_defs,
q->MutableOutputDefs(),
&node_.GetAttributes(),
kOnnxDomain);
qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider);
return true;
}
};
DEFINE_QDQ_CREATOR(MatMul, QDQMatMulTransformer)
} // namespace onnxruntime

60
onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.cc Normal file
View file

@ -0,0 +1,60 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "qdq_op_transformer.h"
#include <string>
#include <vector>
#include "core/graph/graph.h"
#include "core/graph/onnx_protobuf.h"
namespace onnxruntime {
void QDQOperatorTransformer::FillQDQOptionalZeroPoint(const std::vector<const Node*>& qdq_nodes) {
for (const Node* p_node_const : qdq_nodes) {
Node& node = *graph_.GetNode(p_node_const->Index());
std::vector<NodeArg*>& input_defs = node.MutableInputDefs();
constexpr size_t max_input_count = 3;
if (input_defs.size() == max_input_count) {
continue; // zero point is not optional. No need to fill.
}
bool is_default_zp_signed = false;
if (node.OpType() == DQOPTypeName) {
auto input_type = input_defs[0]->TypeAsProto()->tensor_type().elem_type();
is_default_zp_signed = ONNX_NAMESPACE::TensorProto_DataType_INT8 == input_type;
}
const ONNX_NAMESPACE::TensorProto& zp_tensor_proto = is_default_zp_signed ? optional_zero_point_int8_ : optional_zero_point_uint8_;
const ONNX_NAMESPACE::TensorProto* dummy_zp_tensor_proto;
if (!graph_.GetInitializedTensor(zp_tensor_proto.name(), dummy_zp_tensor_proto)) {
graph_.AddInitializedTensor(zp_tensor_proto);
}
input_defs.push_back(&graph_.GetOrCreateNodeArg(zp_tensor_proto.name(), nullptr));
}
}
const ONNX_NAMESPACE::TensorProto QDQOperatorTransformer::optional_zero_point_int8_ = []() {
const char* const name = "855dd0fa-cd7b-4b10-ae5a-df64cabfe1f8";
ONNX_NAMESPACE::TensorProto tensor_proto;
tensor_proto.set_name(name);
tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_INT8);
tensor_proto.set_raw_data(std::vector<int8_t>{0}.data(), sizeof(int8_t));
return tensor_proto;
}();
const ONNX_NAMESPACE::TensorProto QDQOperatorTransformer::optional_zero_point_uint8_ = []() {
const char* const name = "35b188f7-c464-43e3-8692-97ac832bb14a";
ONNX_NAMESPACE::TensorProto tensor_proto;
tensor_proto.set_name(name);
tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
tensor_proto.set_raw_data(std::vector<int8_t>{0}.data(), sizeof(uint8_t));
return tensor_proto;
}();
} // namespace onnxruntime

41
onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.h Normal file
View file

@ -0,0 +1,41 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <vector>
#include "core/graph/onnx_protobuf.h"
namespace onnxruntime {
static const char* const QOPTypeName = "QuantizeLinear";
static const char* const DQOPTypeName = "DequantizeLinear";
class Node;
class Graph;
class QDQOperatorTransformer {
public:
QDQOperatorTransformer(Node& node, Graph& graph) : node_(node), graph_(graph) {}
virtual ~QDQOperatorTransformer() {}
virtual bool Transform(const std::vector<const Node*>& dq_nodes, const std::vector<const Node*>& q_nodes) = 0;
/* Determine whether to keep node_ itself or not.
For operators that support int8, keep node_ and only change its input and output.
Otherwise, node_ will be removed and replaced by a QLinear* version.
*/
virtual bool KeepNode() const {
return false;
}
void FillQDQOptionalZeroPoint(const std::vector<const Node*>& parents);
protected:
Node& node_;
Graph& graph_;
static const ONNX_NAMESPACE::TensorProto optional_zero_point_int8_;
static const ONNX_NAMESPACE::TensorProto optional_zero_point_uint8_;
};
} // namespace onnxruntime

38
onnxruntime/core/optimizer/qdq_transformer/qdq_simple_ops.cc Normal file
View file

@ -0,0 +1,38 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <vector>
#include "core/graph/graph.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
#include "core/optimizer/qdq_transformer/registry.h"
namespace onnxruntime {
class QDQSimpleTransformer : public QDQOperatorTransformer {
public:
QDQSimpleTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {}
bool Transform(const std::vector<const Node*>& parents, const std::vector<const Node*>& children) override {
if (parents.size() != 1 || children.size() != 1) {
return false;
}
FillQDQOptionalZeroPoint(parents);
FillQDQOptionalZeroPoint(children);
graph_.RemoveEdge(parents[0]->Index(), node_.Index(), 0, 0);
graph_.RemoveEdge(node_.Index(), children[0]->Index(), 0, 0);
node_.MutableInputDefs()[0] = graph_.GetNode(parents[0]->Index())->MutableInputDefs()[0];
node_.MutableOutputDefs()[0] = graph_.GetNode(children[0]->Index())->MutableOutputDefs()[0];
return true;
}
bool KeepNode() const override {
return true;
}
};
DEFINE_QDQ_CREATOR(MaxPool, QDQSimpleTransformer)
DEFINE_QDQ_CREATOR(Reshape, QDQSimpleTransformer)
} // namespace onnxruntime

90
onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.cc Normal file
View file

@ -0,0 +1,90 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <deque>
#include <vector>
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
#include "core/optimizer/qdq_transformer/qdq_transformer.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
#include "core/optimizer/qdq_transformer/registry.h"
#include "core/optimizer/utils.h"
using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
namespace onnxruntime {
class QDQTransformerImpl {
public:
QDQTransformerImpl(Graph& graph) noexcept : graph_(graph) {}
void Transform(Node& node) {
// extract DequantizeLinear from parents and QuantizeLinear in children
std::vector<const Node*> parents = graph_utils::FindParentsByType(node, DQOPTypeName);
std::vector<const Node*> children = graph_utils::FindChildrenByType(node, QOPTypeName);
if (parents.size() == 0) {
return;
}
// track dq output edges count
for (auto parent_node : parents) {
if (!dq_output_edges_count_.count(parent_node)) {
dq_output_edges_count_[parent_node] = parent_node->GetOutputEdgesCount();
}
}
std::unique_ptr<QDQOperatorTransformer> op_trans = QDQRegistry::CreateQDQTransformer(node, graph_);
if (op_trans && op_trans->Transform(parents, children)) {
for (auto parent_node : parents) {
dq_output_edges_count_[parent_node]--;
}
UpdateNodesToRemove(parents);
UpdateNodesToRemove(children);
if (!op_trans->KeepNode()) {
nodes_to_remove_.insert(node.Index());
}
}
}
void Finalize(bool& modified) {
for (auto node_idx : nodes_to_remove_) {
graph_utils::RemoveNodeOutputEdges(graph_, *graph_.GetNode(node_idx));
graph_.RemoveNode(node_idx);
}
modified = true;
}
private:
void UpdateNodesToRemove(const std::vector<const Node*>& nodes) {
for (auto node : nodes) {
if (dq_output_edges_count_[node] == 0 && !nodes_to_remove_.count(node->Index())) {
nodes_to_remove_.insert(node->Index());
}
}
}
Graph& graph_;
std::unordered_map<const Node*, size_t> dq_output_edges_count_;
std::set<NodeIndex> nodes_to_remove_;
};
Status QDQTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
QDQTransformerImpl impl(graph);
GraphViewer graph_viewer(graph);
for (auto index : graph_viewer.GetNodesInTopologicalOrder()) {
auto& node = *graph.GetNode(index);
ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger));
if (node.GetExecutionProviderType() == kCpuExecutionProvider) {
impl.Transform(node);
}
}
impl.Finalize(modified);
return Status::OK();
}
} // namespace onnxruntime

24
onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.h Normal file
View file

@ -0,0 +1,24 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include "core/optimizer/graph_transformer.h"
namespace onnxruntime {
/**
@Class QDQTransformer
Transformer that fuse QDQ and fp32 op to quantized op.
*/
class QDQTransformer : public GraphTransformer {
public:
QDQTransformer() noexcept : GraphTransformer("QDQTransformer") {}
private:
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
};
} // namespace onnxruntime

21
onnxruntime/core/optimizer/qdq_transformer/registry.cc Normal file
View file

@ -0,0 +1,21 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "registry.h"
namespace onnxruntime {
DECLARE_QDQ_CREATOR(Conv, QDQConvTransformer)
DECLARE_QDQ_CREATOR(MaxPool, QDQSimpleTransformer)
DECLARE_QDQ_CREATOR(Reshape, QDQSimpleTransformer)
DECLARE_QDQ_CREATOR(Add, QDQBinaryOpTransformer)
DECLARE_QDQ_CREATOR(Mul, QDQBinaryOpTransformer)
DECLARE_QDQ_CREATOR(MatMul, QDQMatMulTransformer)
std::unordered_map<std::string, QDQRegistry::QDQTransformerCreator> QDQRegistry::qdqtransformer_creators_{
REGISTER_QDQ_CREATOR(Conv, QDQConvTransformer),
REGISTER_QDQ_CREATOR(MaxPool, QDQSimpleTransformer),
REGISTER_QDQ_CREATOR(Reshape, QDQSimpleTransformer),
REGISTER_QDQ_CREATOR(Add, QDQBinaryOpTransformer),
REGISTER_QDQ_CREATOR(Mul, QDQBinaryOpTransformer),
REGISTER_QDQ_CREATOR(MatMul, QDQMatMulTransformer),
};
} // namespace onnxruntime

56
onnxruntime/core/optimizer/qdq_transformer/registry.h Normal file
View file

@ -0,0 +1,56 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include "core/graph/graph.h"
#include "core/optimizer/qdq_transformer/qdq_op_transformer.h"
namespace onnxruntime {
class QDQRegistry {
public:
using QDQTransformerCreator = std::function<std::unique_ptr<QDQOperatorTransformer>(Node&, Graph&)>;
static bool Register(const std::string& op_type, QDQTransformerCreator creator) {
if (qdqtransformer_creators_.count(op_type)) {
return false;
}
qdqtransformer_creators_[op_type] = creator;
return true;
}
static std::unique_ptr<QDQOperatorTransformer> CreateQDQTransformer(Node& node, Graph& graph) {
auto it = qdqtransformer_creators_.find(node.OpType());
if (it != qdqtransformer_creators_.end())
return (it->second)(node, graph);
return std::unique_ptr<QDQOperatorTransformer>();
}
private:
static std::unordered_map<std::string, QDQTransformerCreator> qdqtransformer_creators_;
};
#define QDQ_CREATOR_BUILDER_NAME(op_type, Transformer) Register_##op_type##_qdq_##Transformer
#define DECLARE_QDQ_CREATOR(op_type, Transformer) \
std ::pair<std::string, QDQRegistry::QDQTransformerCreator> QDQ_CREATOR_BUILDER_NAME(op_type, Transformer)();
#define DEFINE_QDQ_CREATOR(op_type, Transformer) \
std::pair<std::string, QDQRegistry::QDQTransformerCreator> QDQ_CREATOR_BUILDER_NAME(op_type, Transformer)() { \
return std::pair<std::string, QDQRegistry::QDQTransformerCreator>( \
#op_type, \
[](Node& node, Graph& graph) { return std::make_unique<Transformer>(node, graph); }); \
}
#define REGISTER_QDQ_CREATOR(op_type, Transformer) \
QDQ_CREATOR_BUILDER_NAME(op_type, Transformer) \
()
} // namespace onnxruntime

3
onnxruntime/core/providers/cuda/tensor/compress_impl.cu
View file

@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <cub/cub.cuh>
#include "core/providers/cuda/cu_inc/common.cuh"
#include "core/providers/cuda/cuda_common.h"
@ -10,7 +12,6 @@
#endif
#include "core/providers/cuda/tensor/compress_impl.h"
#include <cub/cub.cuh>
namespace onnxruntime {
namespace cuda {

26
onnxruntime/core/util/math.h
View file

@ -16,18 +16,26 @@
#pragma once
// This is a simple translation from the old Caffe math interfaces. We aim to
// still keep it simple, so all platforms would be able to support it fairly
// easily.
// We include the cblas header here so that we can obtain the macros from cblas.
extern "C" {
#include "core/framework/cblas.h"
}
#include "core/common/common.h"
#include "core/framework/tensor.h"
#ifndef CBLAS_ENUM_DEFINED_H
#define CBLAS_ENUM_DEFINED_H
enum CBLAS_ORDER { CblasRowMajor = 101,
CblasColMajor = 102 };
enum CBLAS_TRANSPOSE {
CblasNoTrans = 111,
CblasTrans = 112,
CblasConjTrans = 113
};
enum CBLAS_UPLO { CblasUpper = 121,
CblasLower = 122 };
enum CBLAS_DIAG { CblasNonUnit = 131,
CblasUnit = 132 };
enum CBLAS_SIDE { CblasLeft = 141,
CblasRight = 142 };
#endif
namespace onnxruntime {
namespace concurrency {
class ThreadPool;

11
onnxruntime/python/tools/quantization/onnx_quantizer.py
View file

@ -573,9 +573,16 @@ class ONNXQuantizer:
packed_bias_scale_initializer = onnx.numpy_helper.from_array(bias_scale_data, quantized_bias_scale_name)
self.model.initializer().extend([packed_bias_scale_initializer])
# update zero initializer
quantized_bias_zp_name = quantized_bias_name + "_zero_point"
bias_zp_data = np.zeros(bias_scale.shape, dtype=np.int32).reshape(-1)
packed_bias_zp_initializer = onnx.numpy_helper.from_array(bias_zp_data, quantized_bias_zp_name)
self.model.initializer().extend([packed_bias_zp_initializer])
assert (bias_name not in self.quantized_value_map)
quantized_value = QuantizedValue(bias_name, quantized_bias_name, quantized_bias_scale_name, "",
QuantizedValueType.Initializer, 0 if bias_scale_data.size > 1 else None)
quantized_value = QuantizedValue(bias_name, quantized_bias_name, quantized_bias_scale_name,
quantized_bias_zp_name, QuantizedValueType.Initializer,
0 if bias_scale_data.size > 1 else None)
self.quantized_value_map[bias_name] = quantized_value
return quantized_bias_name

16
onnxruntime/test/common/tensor_op_test_utils.h
View file

@ -10,6 +10,7 @@
#include "core/common/common.h"
#include "core/common/optional.h"
#include "core/common/type_utils.h"
#include "core/util/math.h"
namespace onnxruntime {
@ -52,7 +53,7 @@ class RandomValueGenerator {
// Random values generated are in the range [min, max).
template <typename TInt>
typename std::enable_if<
std::is_integral<TInt>::value,
std::is_integral<TInt>::value && !utils::IsByteType<TInt>::value,
std::vector<TInt>>::type
Uniform(const std::vector<int64_t>& dims, TInt min, TInt max) {
std::vector<TInt> val(detail::SizeFromDims(dims));
@ -63,6 +64,19 @@ class RandomValueGenerator {
return val;
}
template <typename TByte>
typename std::enable_if<
utils::IsByteType<TByte>::value,
std::vector<TByte>>::type
Uniform(const std::vector<int64_t>& dims, TByte min, TByte max) {
std::vector<TByte> val(detail::SizeFromDims(dims));
std::uniform_int_distribution<int32_t> distribution(min, max - 1);
for (size_t i = 0; i < val.size(); ++i) {
val[i] = static_cast<TByte>(distribution(generator_));
}
return val;
}
// Gaussian distribution for float
template <typename TFloat>
typename std::enable_if<

84
onnxruntime/test/optimizer/graph_transform_test_builder.cc Normal file
View file

@ -0,0 +1,84 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <functional>
#include <string>
#include <vector>
#include "core/graph/model.h"
#include "core/session/inference_session.h"
#include "test/compare_ortvalue.h"
#include "test/test_environment.h"
#include "test/util/include/inference_session_wrapper.h"
#include "graph_transform_test_builder.h"
namespace onnxruntime {
namespace test {
void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& build_test_case,
const std::function<void(InferenceSessionWrapper& session)>& check_transformed_graph,
TransformerLevel baseline_level,
TransformerLevel target_level,
int opset_version) {
// Build the model for this test.
std::unordered_map<std::string, int> domain_to_version;
domain_to_version[kOnnxDomain] = opset_version;
domain_to_version[kMSDomain] = 1;
Model model("TransformerTester", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(),
domain_to_version, {}, DefaultLoggingManager().DefaultLogger());
Graph& graph = model.MainGraph();
ModelTestBuilder helper(graph);
build_test_case(helper);
ASSERT_TRUE(model.MainGraph().Resolve().IsOK());
// Serialize the model to a string.
std::string model_data;
model.ToProto().SerializeToString(&model_data);
auto run_model = [&](TransformerLevel level, std::vector<OrtValue>& fetches) {
SessionOptions session_options;
session_options.graph_optimization_level = level;
InferenceSessionWrapper session{session_options, GetEnvironment()};
ASSERT_TRUE(session.Load(model_data.data(), static_cast<int>(model_data.size())).IsOK());
ASSERT_TRUE(session.Initialize().IsOK());
RunOptions run_options;
auto status = session.Run(run_options,
helper.feeds_,
helper.output_names_,
&fetches);
if (!status.IsOK()) {
std::cout << "Run failed with status message: " << status.ErrorMessage() << std::endl;
}
ASSERT_TRUE(status.IsOK());
if (level == target_level) {
check_transformed_graph(session);
}
};
std::vector<OrtValue> baseline_fetches;
run_model(baseline_level, baseline_fetches);
std::vector<OrtValue> target_fetches;
run_model(target_level, target_fetches);
size_t num_outputs = baseline_fetches.size();
ASSERT_TRUE(num_outputs == target_fetches.size());
for (size_t i = 0; i < num_outputs; i++) {
double per_sample_tolerance = 0.0;
double relative_per_sample_tolerance = 0.0;
std::pair<COMPARE_RESULT, std::string> ret =
CompareOrtValue(target_fetches[i],
baseline_fetches[i],
per_sample_tolerance,
relative_per_sample_tolerance,
false);
EXPECT_EQ(ret.first, COMPARE_RESULT::SUCCESS) << ret.second;
}
}
} // namespace test
} // namespace onnxruntime

246
onnxruntime/test/optimizer/graph_transform_test_builder.h Normal file
View file

@ -0,0 +1,246 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <functional>
#include <type_traits>
#include <vector>
#include "core/common/type_utils.h"
#include "core/graph/graph.h"
#include "core/framework/framework_common.h"
#include "core/optimizer/graph_transformer_level.h"
#include "core/graph/onnx_protobuf.h"
#include "test/framework/test_utils.h"
#include "test/common/tensor_op_test_utils.h"
#include "test/framework/test_utils.h"
#include "test/util/include/inference_session_wrapper.h"
namespace onnxruntime {
namespace test {
template <typename T>
struct IsTypeQuantLinearCompatible : utils::IsByteType<T> {};
template <typename T>
struct IsTypeDequantLinearCompatible : utils::IsByteType<T> {};
template <>
struct IsTypeDequantLinearCompatible<int32_t> : std::true_type {};
class ModelTestBuilder {
public:
ModelTestBuilder(Graph& graph) : graph_(graph) {
}
template <typename T>
NodeArg* MakeInput(const std::vector<int64_t>& shape, T min, T max) {
ONNX_NAMESPACE::TypeProto type_proto;
type_proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType<T>());
for (auto& dim : shape) {
type_proto.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim);
}
OrtValue input_value;
CreateMLValue<T>(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault),
shape,
rand_gen_.Uniform<T>(shape, min, max),
&input_value);
std::string name = graph_.GenerateNodeArgName("input");
feeds_.insert(std::make_pair(name, input_value));
return &graph_.GetOrCreateNodeArg(name, &type_proto);
}
NodeArg* MakeOutput() {
std::string name = graph_.GenerateNodeArgName("output");
output_names_.push_back(name);
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
NodeArg* MakeIntermediate() {
std::string name = graph_.GenerateNodeArgName("node");
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
template <typename T>
NodeArg* MakeInitializer(const std::vector<int64_t>& shape, const std::vector<T>& data) {
std::string name = graph_.GenerateNodeArgName("constant");
ONNX_NAMESPACE::TensorProto tensor_proto;
tensor_proto.set_name(name);
tensor_proto.set_data_type(utils::ToTensorProtoElementType<T>());
tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T));
for (auto& dim : shape) {
tensor_proto.add_dims(dim);
}
graph_.AddInitializedTensor(tensor_proto);
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
template <typename T>
NodeArg* MakeInitializer(const std::vector<int64_t>& shape, T min, T max) {
return MakeInitializer<T>(shape, rand_gen_.Uniform<T>(shape, min, max));
}
template <typename T>
NodeArg* MakeScalarInitializer(T data) {
return MakeInitializer({}, std::vector<T>{data});
}
template <typename T>
NodeArg* Make1DInitializer(const std::vector<T>& data) {
return MakeInitializer({static_cast<int64_t>(data.size())}, data);
}
Node& AddNode(const std::string& op_type,
const std::vector<NodeArg*>& input_args,
const std::vector<NodeArg*>& output_args,
const std::string& domain = "") {
return graph_.AddNode(graph_.GenerateNodeName("node"),
op_type,
"description",
input_args,
output_args,
nullptr,
domain);
}
Node& AddConvNode(NodeArg* input_arg,
NodeArg* weights_arg,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(weights_arg);
return AddNode("Conv", input_args, {output_arg});
}
template <typename T>
typename std::enable_if<IsTypeQuantLinearCompatible<T>::value, Node&>::type
AddQuantizeLinearNode(NodeArg* input_arg,
float input_scale,
T input_zero_point,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<T>(input_zero_point));
return AddNode("QuantizeLinear", input_args, {output_arg});
}
Node& AddQuantizeLinearNode(NodeArg* input_arg,
float input_scale,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
return AddNode("QuantizeLinear", input_args, {output_arg});
}
template <typename T>
typename std::enable_if<IsTypeDequantLinearCompatible<T>::value, Node&>::type
AddDequantizeLinearNode(NodeArg* input_arg,
float input_scale,
T input_zero_point,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<T>(input_zero_point));
return AddNode("DequantizeLinear", input_args, {output_arg});
}
template <typename T>
typename std::enable_if<IsTypeDequantLinearCompatible<T>::value, Node&>::type
AddDequantizeLinearNode(NodeArg* input_arg,
float input_scale,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
return AddNode("DequantizeLinear", input_args, {output_arg});
}
template <typename TWeight>
Node& AddQLinearConvNode(NodeArg* input_arg,
float input_scale,
uint8_t input_zero_point,
NodeArg* weight_arg,
float weights_scale,
TWeight weights_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
std::vector<NodeArg*> input_args{input_arg};
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input_zero_point));
input_args.push_back(weight_arg);
input_args.push_back(MakeScalarInitializer<float>(weights_scale));
input_args.push_back(MakeScalarInitializer<TWeight>(weights_zero_point));
input_args.push_back(MakeScalarInitializer<float>(output_scale));
input_args.push_back(MakeScalarInitializer<TWeight>(output_zero_point));
return AddNode("QLinearConv", input_args, {output_arg});
}
Node& AddQLinearBinaryNode(const std::string& op_type,
NodeArg* input1_arg,
float input1_scale,
uint8_t input1_zero_point,
NodeArg* input2_arg,
float input2_scale,
uint8_t input2_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
std::vector<NodeArg*> input_args;
input_args.push_back(input1_arg);
input_args.push_back(MakeScalarInitializer<float>(input1_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input1_zero_point));
input_args.push_back(input2_arg);
input_args.push_back(MakeScalarInitializer<float>(input2_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input2_zero_point));
input_args.push_back(MakeScalarInitializer<float>(output_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(output_zero_point));
return AddNode(op_type, input_args, {output_arg}, kMSDomain);
}
Node& AddQLinearActivationNode(const std::string& op_type,
NodeArg* input_arg,
float input_scale,
uint8_t input_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input_zero_point));
input_args.push_back(MakeScalarInitializer<float>(output_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(output_zero_point));
return AddNode(op_type, input_args, {output_arg}, kMSDomain);
}
Graph& graph_;
NameMLValMap feeds_;
std::vector<std::string> output_names_;
RandomValueGenerator rand_gen_{optional<RandomValueGenerator::RandomSeedType>{2345}};
};
void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& build_test_case,
const std::function<void(InferenceSessionWrapper& session)>& check_transformed_graph,
TransformerLevel baseline_level,
TransformerLevel target_level,
int opset_version = 12);
} // namespace test
} // namespace onnxruntime

655
onnxruntime/test/optimizer/nhwc_transformer_test.cc
View file

@ -2,17 +2,12 @@
// Licensed under the MIT License.
#include <random>
#include "core/graph/model.h"
#include "core/graph/onnx_protobuf.h"
#include "core/mlas/inc/mlas.h"
#include "core/session/environment.h"
#include "core/session/inference_session.h"
#include "test/compare_ortvalue.h"
#include "test/test_environment.h"
#include "test/framework/test_utils.h"
#include "test/util/include/inference_session_wrapper.h"
#include <vector>
#include "gtest/gtest.h"
#include "graph_transform_test_builder.h"
#include "core/graph/graph.h"
namespace onnxruntime {
namespace test {
@ -30,282 +25,25 @@ struct NhwcWeightsRange<int8_t> {
static constexpr int8_t max_value = +63;
};
struct NhwcTestHelper {
NhwcTestHelper(Graph& graph) : graph_(graph) {
}
template <typename T>
NodeArg* MakeInput(const std::vector<int64_t>& shape, const ONNX_NAMESPACE::TypeProto& type_proto) {
OrtValue input_value;
CreateMLValue<T>(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), shape,
FillRandomData<T>(shape, 0, 31), &input_value);
std::string name = graph_.GenerateNodeArgName("input");
feeds_.insert(std::make_pair(name, input_value));
return &graph_.GetOrCreateNodeArg(name, &type_proto);
}
template <typename T>
NodeArg* MakeInput(const std::vector<int64_t>& shape) {
ONNX_NAMESPACE::TypeProto type_proto;
type_proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType<T>());
for (auto& dim : shape) {
type_proto.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim);
}
return MakeInput<T>(shape, type_proto);
}
NodeArg* MakeOutput() {
std::string name = graph_.GenerateNodeArgName("output");
output_names_.push_back(name);
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
NodeArg* MakeIntermediate() {
std::string name = graph_.GenerateNodeArgName("node");
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
template <typename T>
NodeArg* MakeInitializer(const std::vector<int64_t>& shape, const std::vector<T>& data) {
std::string name = graph_.GenerateNodeArgName("constant");
ONNX_NAMESPACE::TensorProto tensor_proto;
tensor_proto.set_name(name);
tensor_proto.set_data_type(utils::ToTensorProtoElementType<T>());
tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T));
for (auto& dim : shape) {
tensor_proto.add_dims(dim);
}
graph_.AddInitializedTensor(tensor_proto);
return &graph_.GetOrCreateNodeArg(name, nullptr);
}
template <typename T>
NodeArg* MakeInitializer(const std::vector<int64_t>& shape, int32_t min_value, int32_t max_value) {
return MakeInitializer<T>(shape, FillRandomData<T>(shape, min_value, max_value));
}
template <typename T>
NodeArg* MakeScalarInitializer(T data) {
return MakeInitializer({}, std::vector<T>{data});
}
template <typename T>
NodeArg* Make1DInitializer(const std::vector<T>& data) {
return MakeInitializer({static_cast<int64_t>(data.size())}, data);
}
template <typename T>
NodeArg* MakeWeightsInitializer(const std::vector<int64_t>& shape) {
return MakeInitializer<T>(shape, NhwcWeightsRange<T>::min_value, NhwcWeightsRange<T>::max_value);
}
Node& AddNode(const std::string& op_type,
const std::vector<NodeArg*>& input_args,
const std::vector<NodeArg*>& output_args,
const std::string& domain = "") {
return graph_.AddNode(graph_.GenerateNodeName("node"),
op_type,
"description",
input_args,
output_args,
nullptr,
domain);
}
Node& AddQLinearConvNode(NodeArg* input_arg,
NodeArg* input_scale_arg,
NodeArg* input_zero_point_arg,
NodeArg* weights_arg,
NodeArg* weights_scale_arg,
NodeArg* weights_zero_point_arg,
NodeArg* output_arg,
NodeArg* output_scale,
NodeArg* output_zero_point) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(input_scale_arg);
input_args.push_back(input_zero_point_arg);
input_args.push_back(weights_arg);
input_args.push_back(weights_scale_arg);
input_args.push_back(weights_zero_point_arg);
input_args.push_back(output_scale);
input_args.push_back(output_zero_point);
return AddNode("QLinearConv", input_args, {output_arg});
}
template <typename TWeight>
Node& AddQLinearConvNode(NodeArg* input_arg,
float input_scale,
uint8_t input_zero_point,
const std::vector<int64_t>& weights_shape,
float weights_scale,
TWeight weights_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
return AddQLinearConvNode(input_arg,
MakeScalarInitializer<float>(input_scale),
MakeScalarInitializer<uint8_t>(input_zero_point),
MakeWeightsInitializer<TWeight>(weights_shape),
MakeScalarInitializer<float>(weights_scale),
MakeScalarInitializer<TWeight>(weights_zero_point),
output_arg,
MakeScalarInitializer<float>(output_scale),
MakeScalarInitializer<TWeight>(output_zero_point));
}
Node& AddQLinearBinaryNode(const std::string& op_type,
NodeArg* input1_arg,
float input1_scale,
uint8_t input1_zero_point,
NodeArg* input2_arg,
float input2_scale,
uint8_t input2_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
std::vector<NodeArg*> input_args;
input_args.push_back(input1_arg);
input_args.push_back(MakeScalarInitializer<float>(input1_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input1_zero_point));
input_args.push_back(input2_arg);
input_args.push_back(MakeScalarInitializer<float>(input2_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input2_zero_point));
input_args.push_back(MakeScalarInitializer<float>(output_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(output_zero_point));
return AddNode(op_type, input_args, {output_arg}, kMSDomain);
}
Node& AddQLinearActivationNode(const std::string& op_type,
NodeArg* input_arg,
float input_scale,
uint8_t input_zero_point,
NodeArg* output_arg,
float output_scale,
uint8_t output_zero_point) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input_zero_point));
input_args.push_back(MakeScalarInitializer<float>(output_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(output_zero_point));
return AddNode(op_type, input_args, {output_arg}, kMSDomain);
}
Node& AddDequantizeLinearNode(NodeArg* input_arg,
float input_scale,
uint8_t input_zero_point,
NodeArg* output_arg) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(MakeScalarInitializer<float>(input_scale));
input_args.push_back(MakeScalarInitializer<uint8_t>(input_zero_point));
return AddNode("DequantizeLinear", input_args, {output_arg});
}
template <typename T>
std::vector<T> FillRandomData(size_t count, int32_t min_value, int32_t max_value) {
std::vector<T> random_data;
random_data.resize(count);
std::uniform_int_distribution<int32_t> distribution(min_value, max_value);
for (size_t n = 0; n < count; n++) {
random_data[n] = static_cast<T>(distribution(generator_));
}
return random_data;
}
template <typename T>
std::vector<T> FillRandomData(const std::vector<int64_t>& shape, int32_t min_value, int32_t max_value) {
int64_t num_elements = std::accumulate(shape.begin(), shape.end(), int64_t(1), std::multiplies<int64_t>{});
return FillRandomData<T>(static_cast<size_t>(num_elements), min_value, max_value);
}
Graph& graph_;
NameMLValMap feeds_;
std::vector<std::string> output_names_;
std::default_random_engine generator_{2345};
};
void NhwcTransformerTester(const std::function<void(NhwcTestHelper& helper)>& build_test_case,
const std::function<void(InferenceSessionWrapper& session)>& check_nhwc_graph,
int opset_version = 12) {
// Build the model for this test.
std::unordered_map<std::string, int> domain_to_version;
domain_to_version[kOnnxDomain] = opset_version;
domain_to_version[kMSDomain] = 1;
Model model("nhwc", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(),
domain_to_version, {}, DefaultLoggingManager().DefaultLogger());
NhwcTestHelper helper(model.MainGraph());
build_test_case(helper);
ASSERT_TRUE(model.MainGraph().Resolve().IsOK());
// Serialize the model to a string.
std::string model_data;
model.ToProto().SerializeToString(&model_data);
auto run_model = [&](TransformerLevel level, std::vector<OrtValue>& fetches) {
SessionOptions session_options;
session_options.graph_optimization_level = level;
session_options.session_logid = "NhwcTransformerTests";
InferenceSessionWrapper session{session_options, GetEnvironment()};
ASSERT_TRUE(session.Load(model_data.data(), static_cast<int>(model_data.size())).IsOK());
ASSERT_TRUE(session.Initialize().IsOK());
RunOptions run_options;
auto status = session.Run(run_options, helper.feeds_, helper.output_names_, &fetches);
if (!status.IsOK()) {
std::cout << "Run failed with status message: " << status.ErrorMessage() << std::endl;
}
ASSERT_TRUE(status.IsOK());
if (level == TransformerLevel::Level3) {
check_nhwc_graph(session);
}
};
std::vector<OrtValue> level2_fetches;
run_model(TransformerLevel::Level2, level2_fetches);
std::vector<OrtValue> level3_fetches;
run_model(TransformerLevel::Level3, level3_fetches);
size_t num_outputs = level2_fetches.size();
ASSERT_TRUE(num_outputs == level3_fetches.size());
for (size_t i = 0; i < num_outputs; i++) {
double per_sample_tolerance = 0.0;
double relative_per_sample_tolerance = 0.0;
std::pair<COMPARE_RESULT, std::string> ret =
CompareOrtValue(level3_fetches[i],
level2_fetches[i],
per_sample_tolerance,
relative_per_sample_tolerance,
false);
EXPECT_EQ(ret.first, COMPARE_RESULT::SUCCESS) << ret.second;
}
template <typename T>
NodeArg* NhwcMakeInitializer(ModelTestBuilder& builder, const std::vector<int64_t>& shape) {
return builder.MakeInitializer<T>(shape,
NhwcWeightsRange<T>::min_value,
NhwcWeightsRange<T>::max_value);
}
#ifndef DISABLE_CONTRIB_OPS
TEST(NhwcTransformerTests, Conv) {
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>(input_shape);
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>(input_shape, 0, 31);
auto* output_arg = builder.MakeOutput();
auto* weight_arg = NhwcMakeInitializer<uint8_t>(builder, weights_shape);
helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
weights_shape, .02f, 126,
output_arg, .37f, 131);
builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
weight_arg, .02f, 126,
output_arg, .37f, 131);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -314,7 +52,10 @@ TEST(NhwcTransformerTests, Conv) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
};
// Test the basic case of a single 1D/2D/3D convolution.
@ -324,15 +65,18 @@ TEST(NhwcTransformerTests, Conv) {
}
TEST(NhwcTransformerTests, ConvDequantizeLinear) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({1, 12, 37});
auto* conv_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({1, 12, 37}, 0, 31);
auto* conv_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
auto* weight_arg = NhwcMakeInitializer<uint8_t>(builder, {32, 12, 5});
helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{32, 12, 5}, .02f, 126,
conv_output_arg, .37f, 131);
helper.AddDequantizeLinearNode(conv_output_arg, .37f, 131, output_arg);
builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
weight_arg, .02f, 126,
conv_output_arg, .37f, 131);
builder.AddDequantizeLinearNode<uint8_t>(conv_output_arg,
.37f, 131,
output_arg);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -344,28 +88,38 @@ TEST(NhwcTransformerTests, ConvDequantizeLinear) {
// QLinearConv followed by only DequantizeLinear will remain as the ONNX
// version of the operator to avoid adding unnecessary Transpose nodes to
// the graph.
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
TEST(NhwcTransformerTests, ConvBlockBinary) {
auto test_case = [&](const std::string& binary_op_type) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({1, 23, 13, 13});
auto* conv1_output_arg = helper.MakeIntermediate();
auto* conv2_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({1, 23, 13, 13}, 0, 31);
auto* conv1_output_arg = builder.MakeIntermediate();
auto* conv2_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
auto* conv1_weight_arg = builder.MakeInitializer<uint8_t>({30, 23, 3, 3},
NhwcWeightsRange<uint8_t>::min_value,
NhwcWeightsRange<uint8_t>::max_value);
Node& conv1_node = helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{30, 23, 3, 3}, .02f, 126,
conv1_output_arg, .37f, 131);
auto* conv2_weight_arg = builder.MakeInitializer<uint8_t>({30, 23, 1, 1},
NhwcWeightsRange<uint8_t>::min_value,
NhwcWeightsRange<uint8_t>::max_value);
Node& conv1_node = builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv1_output_arg, .37f, 131);
conv1_node.AddAttribute("pads", std::vector<int64_t>{1, 1, 1, 1});
helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{30, 23, 1, 1}, .015f, 129,
conv2_output_arg, .37f, 131);
helper.AddQLinearBinaryNode(binary_op_type,
conv1_output_arg, .37f, 131,
conv2_output_arg, .37f, 131,
output_arg, .43f, 126);
builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv2_weight_arg, .015f, 129,
conv2_output_arg, .37f, 131);
builder.AddQLinearBinaryNode(binary_op_type,
conv1_output_arg, .37f, 131,
conv2_output_arg, .37f, 131,
output_arg, .43f, 126);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -374,7 +128,10 @@ TEST(NhwcTransformerTests, ConvBlockBinary) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
};
std::vector<std::string> activation_op_types{"QLinearAdd", "QLinearMul"};
@ -385,15 +142,16 @@ TEST(NhwcTransformerTests, ConvBlockBinary) {
TEST(NhwcTransformerTests, ConvMaxPool) {
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>(input_shape);
auto* conv_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>(input_shape, 0, 31);
auto* conv_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
auto* conv_weight_arg = NhwcMakeInitializer<uint8_t>(builder, weights_shape);
helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
weights_shape, .02f, 126,
conv_output_arg, .37f, 131);
Node& pool_node = helper.AddNode("MaxPool", {conv_output_arg}, {output_arg});
builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv_weight_arg, .02f, 126,
conv_output_arg, .37f, 131);
Node& pool_node = builder.AddNode("MaxPool", {conv_output_arg}, {output_arg});
std::vector<int64_t> pads((weights_shape.size() - 2) * 2, 1);
pool_node.AddAttribute("pads", pads);
std::vector<int64_t> kernel_shape(weights_shape.size() - 2, 3);
@ -407,7 +165,10 @@ TEST(NhwcTransformerTests, ConvMaxPool) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
};
// Test the basic case of a single 1D/2D/3D convolution.
@ -417,16 +178,17 @@ TEST(NhwcTransformerTests, ConvMaxPool) {
}
TEST(NhwcTransformerTests, ConvMaxPoolIndexTensor) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({1, 16, 17, 17});
auto* conv_output_arg = helper.MakeIntermediate();
auto* index_output_arg = helper.MakeOutput();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({1, 16, 17, 17}, 0, 31);
auto* conv_output_arg = builder.MakeIntermediate();
auto* index_output_arg = builder.MakeOutput();
auto* output_arg = builder.MakeOutput();
auto* conv_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {16, 16, 3, 3});
helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{16, 16, 3, 3}, .02f, 126,
conv_output_arg, .37f, 131);
Node& pool_node = helper.AddNode("MaxPool", {conv_output_arg}, {output_arg, index_output_arg});
builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv_weight_arg, .02f, 126,
conv_output_arg, .37f, 131);
Node& pool_node = builder.AddNode("MaxPool", {conv_output_arg}, {output_arg, index_output_arg});
pool_node.AddAttribute("kernel_shape", std::vector<int64_t>{3, 3});
};
@ -438,32 +200,37 @@ TEST(NhwcTransformerTests, ConvMaxPoolIndexTensor) {
};
// Test that MaxPool using the optional index tensor is not converted to NhwcMaxPool.
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
TEST(NhwcTransformerTests, ConvGlobalAveragePool) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({1, 23, 13, 13});
auto* conv1_output_arg = helper.MakeIntermediate();
auto* conv2_output_arg = helper.MakeIntermediate();
auto* gavgpool1_output_arg = helper.MakeIntermediate();
auto* gavgpool2_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({1, 23, 13, 13}, 0, 31);
auto* conv1_output_arg = builder.MakeIntermediate();
auto* conv2_output_arg = builder.MakeIntermediate();
auto* gavgpool1_output_arg = builder.MakeIntermediate();
auto* gavgpool2_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
auto* conv1_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {30, 23, 3, 3});
auto* conv2_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {16, 30, 1, 1});
Node& conv1_node = helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{30, 23, 3, 3}, .02f, 126,
conv1_output_arg, .37f, 131);
Node& conv1_node = builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv1_output_arg, .37f, 131);
conv1_node.AddAttribute("pads", std::vector<int64_t>{1, 1, 1, 1});
helper.AddQLinearActivationNode("QLinearGlobalAveragePool",
conv1_output_arg, .37f, 131,
gavgpool1_output_arg, .43f, 111);
helper.AddQLinearConvNode<uint8_t>(gavgpool1_output_arg, .43f, 111,
{16, 30, 1, 1}, .015f, 129,
builder.AddQLinearActivationNode("QLinearGlobalAveragePool",
conv1_output_arg, .37f, 131,
gavgpool1_output_arg, .43f, 111);
builder.AddQLinearConvNode<uint8_t>(gavgpool1_output_arg, .43f, 111,
conv2_weight_arg, .015f, 129,
conv2_output_arg, .37f, 131);
helper.AddQLinearActivationNode("QLinearGlobalAveragePool",
conv2_output_arg, .37f, 131,
gavgpool2_output_arg, .37f, 131);
helper.AddDequantizeLinearNode(gavgpool2_output_arg, .37f, 131, output_arg);
builder.AddQLinearActivationNode("QLinearGlobalAveragePool",
conv2_output_arg, .37f, 131,
gavgpool2_output_arg, .37f, 131);
builder.AddDequantizeLinearNode<uint8_t>(gavgpool2_output_arg, .37f, 131, output_arg);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -472,35 +239,42 @@ TEST(NhwcTransformerTests, ConvGlobalAveragePool) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
TEST(NhwcTransformerTests, ConvSplit) {
for (int64_t axis = -4LL; axis < 4; axis++) {
auto build_test_case = [&, axis](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({2, 23, 16, 16});
auto* conv_output_arg = helper.MakeIntermediate();
auto* split_output1_arg = helper.MakeIntermediate();
auto* split_output2_arg = helper.MakeIntermediate();
auto* qladd_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&, axis](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({2, 23, 16, 16}, 0, 31);
auto* conv_output_arg = builder.MakeIntermediate();
auto* split_output1_arg = builder.MakeIntermediate();
auto* split_output2_arg = builder.MakeIntermediate();
auto* qladd_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
const int64_t conv1_output_channels = 32;
Node& conv_node = helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{conv1_output_channels, 23, 3, 3}, .02f, 126,
conv_output_arg, .37f, 131);
auto* conv1_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {conv1_output_channels, 23, 3, 3});
Node& conv_node = builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv_output_arg, .37f, 131);
conv_node.AddAttribute("pads", std::vector<int64_t>{1, 1, 1, 1});
Node& split_node = helper.AddNode("Split", {conv_output_arg}, {split_output1_arg, split_output2_arg});
Node& split_node = builder.AddNode("Split", {conv_output_arg}, {split_output1_arg, split_output2_arg});
split_node.AddAttribute("axis", static_cast<int64_t>(axis));
helper.AddQLinearBinaryNode("QLinearAdd",
split_output1_arg, .37f, 131,
split_output2_arg, .37f, 131,
qladd_output_arg, .43f, 126);
builder.AddQLinearBinaryNode("QLinearAdd",
split_output1_arg, .37f, 131,
split_output2_arg, .37f, 131,
qladd_output_arg, .43f, 126);
const int64_t channels_after_split =
(axis == 1 || axis == -3) ? conv1_output_channels / 2 : conv1_output_channels;
helper.AddQLinearConvNode<uint8_t>(qladd_output_arg, .43f, 126,
{17, channels_after_split, 3, 3}, .02f, 126,
output_arg, .37f, 131);
(axis == 1 || axis == -3) ? conv1_output_channels / 2 : conv1_output_channels;
auto* conv2_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {17, channels_after_split, 3, 3});
builder.AddQLinearConvNode<uint8_t>(qladd_output_arg, .43f, 126,
conv2_weight_arg, .02f, 126,
output_arg, .37f, 131);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -509,32 +283,38 @@ TEST(NhwcTransformerTests, ConvSplit) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
}
TEST(NhwcTransformerTests, ConvPad) {
std::vector<std::string> pad_modes{"constant", "reflect", "edge"};
for (const auto& mode : pad_modes) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input_arg = helper.MakeInput<uint8_t>({1, 23, 13, 13});
auto* conv1_output_arg = helper.MakeIntermediate();
auto* pads_const = helper.MakeScalarInitializer<uint8_t>(131);
auto* pads_arg = helper.Make1DInitializer<int64_t>({0, 0, 1, 2, 0, 0, 3, 4});
auto* pad_output_arg = helper.MakeIntermediate();
auto* conv2_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<uint8_t>({1, 23, 13, 13}, 0, 31);
auto* conv1_output_arg = builder.MakeIntermediate();
auto* pads_const = builder.MakeScalarInitializer<uint8_t>(131);
auto* pads_arg = builder.Make1DInitializer<int64_t>({0, 0, 1, 2, 0, 0, 3, 4});
auto* pad_output_arg = builder.MakeIntermediate();
auto* conv2_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
Node& conv1_node = helper.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
{30, 23, 3, 3}, .02f, 126,
conv1_output_arg, .37f, 131);
auto* conv1_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {30, 23, 3, 3});
Node& conv1_node = builder.AddQLinearConvNode<uint8_t>(input_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv1_output_arg, .37f, 131);
conv1_node.AddAttribute("pads", std::vector<int64_t>{1, 1, 1, 1});
Node& pad_node = helper.AddNode("Pad", {conv1_output_arg, pads_arg, pads_const}, {pad_output_arg});
Node& pad_node = builder.AddNode("Pad", {conv1_output_arg, pads_arg, pads_const}, {pad_output_arg});
pad_node.AddAttribute("mode", mode);
helper.AddQLinearConvNode<uint8_t>(pad_output_arg, .37f, 131,
{16, 30, 3, 3}, .015f, 129,
conv2_output_arg, .37f, 131);
helper.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, output_arg);
auto* conv2_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {16, 30, 3, 3});
builder.AddQLinearConvNode<uint8_t>(pad_output_arg, .37f, 131,
conv2_weight_arg, .015f, 129,
conv2_output_arg, .37f, 131);
builder.AddDequantizeLinearNode<uint8_t>(conv2_output_arg, .37f, 131, output_arg);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -543,59 +323,65 @@ TEST(NhwcTransformerTests, ConvPad) {
EXPECT_EQ(op_to_count["Transpose"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
}
TEST(NhwcTransformerTests, ConvBlockActivation) {
auto test_case = [&](uint32_t extra_edges) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input1_arg = helper.MakeInput<uint8_t>({1, 10, 13, 13});
auto* input2_arg = helper.MakeInput<uint8_t>({1, 13, 13, 13});
auto* concat_arg = helper.MakeIntermediate();
auto* conv1_output_arg = helper.MakeIntermediate();
auto* conv2_output_arg = helper.MakeIntermediate();
auto* act1_output_arg = helper.MakeIntermediate();
auto* act2_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<uint8_t>({1, 10, 13, 13}, 0, 31);
auto* input2_arg = builder.MakeInput<uint8_t>({1, 13, 13, 13}, 0, 31);
auto* concat_arg = builder.MakeIntermediate();
auto* conv1_output_arg = builder.MakeIntermediate();
auto* conv2_output_arg = builder.MakeIntermediate();
auto* act1_output_arg = builder.MakeIntermediate();
auto* act2_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
// Create a convolution input that isn't directly a graph input.
Node& concat_node = helper.AddNode("Concat", {input1_arg, input2_arg}, {concat_arg});
Node& concat_node = builder.AddNode("Concat", {input1_arg, input2_arg}, {concat_arg});
concat_node.AddAttribute("axis", static_cast<int64_t>(1));
Node& conv1_node = helper.AddQLinearConvNode<uint8_t>(concat_arg, .01f, 135,
{30, 23, 3, 3}, .02f, 126,
conv1_output_arg, .37f, 131);
auto* conv1_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {30, 23, 3, 3});
Node& conv1_node = builder.AddQLinearConvNode<uint8_t>(concat_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv1_output_arg, .37f, 131);
conv1_node.AddAttribute("pads", std::vector<int64_t>{1, 1, 1, 1});
helper.AddQLinearActivationNode("QLinearSigmoid",
conv1_output_arg, .37f, 131,
act1_output_arg, .37f, 131);
helper.AddQLinearConvNode<uint8_t>(concat_arg, .01f, 135,
{30, 23, 1, 1}, .015f, 129,
conv2_output_arg, .37f, 131);
helper.AddQLinearActivationNode("QLinearLeakyRelu",
conv2_output_arg, .37f, 131,
act2_output_arg, .37f, 131);
helper.AddQLinearBinaryNode("QLinearAdd",
act1_output_arg, .37f, 131,
act2_output_arg, .37f, 131,
output_arg, .39f, 126);
builder.AddQLinearActivationNode("QLinearSigmoid",
conv1_output_arg, .37f, 131,
act1_output_arg, .37f, 131);
auto* conv2_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {30, 23, 1, 1});
builder.AddQLinearConvNode<uint8_t>(concat_arg, .01f, 135,
conv2_weight_arg, .015f, 129,
conv2_output_arg, .37f, 131);
builder.AddQLinearActivationNode("QLinearLeakyRelu",
conv2_output_arg, .37f, 131,
act2_output_arg, .37f, 131);
builder.AddQLinearBinaryNode("QLinearAdd",
act1_output_arg, .37f, 131,
act2_output_arg, .37f, 131,
output_arg, .39f, 126);
// Create extra uses of the various NodeArgs to exercise the transformer.
if ((extra_edges & 1) != 0) {
helper.AddDequantizeLinearNode(concat_arg, .01f, 135, helper.MakeOutput());
builder.AddDequantizeLinearNode<uint8_t>(concat_arg, .01f, 135, builder.MakeOutput());
}
if ((extra_edges & 2) != 0) {
helper.AddDequantizeLinearNode(conv1_output_arg, .37f, 131, helper.MakeOutput());
builder.AddDequantizeLinearNode<uint8_t>(conv1_output_arg, .37f, 131, builder.MakeOutput());
}
if ((extra_edges & 4) != 0) {
helper.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, helper.MakeOutput());
builder.AddDequantizeLinearNode<uint8_t>(conv2_output_arg, .37f, 131, builder.MakeOutput());
}
if ((extra_edges & 8) != 0) {
helper.AddDequantizeLinearNode(act1_output_arg, .37f, 131, helper.MakeOutput());
builder.AddDequantizeLinearNode<uint8_t>(act1_output_arg, .37f, 131, builder.MakeOutput());
}
if ((extra_edges & 16) != 0) {
helper.AddDequantizeLinearNode(act2_output_arg, .37f, 131, helper.MakeOutput());
builder.AddDequantizeLinearNode<uint8_t>(act2_output_arg, .37f, 131, builder.MakeOutput());
}
};
@ -604,7 +390,10 @@ TEST(NhwcTransformerTests, ConvBlockActivation) {
EXPECT_EQ(op_to_count["com.microsoft.QLinearConv"], 2);
};
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
};
// Add extra uses of the edges that cause the transformer to insert additional
@ -615,24 +404,27 @@ TEST(NhwcTransformerTests, ConvBlockActivation) {
}
TEST(NhwcTransformerTests, ConvMixTensorRanks) {
auto build_test_case = [&](NhwcTestHelper& helper) {
auto* input1_arg = helper.MakeInput<uint8_t>({1, 10, 7});
auto* input2_arg = helper.MakeInput<uint8_t>({1, 12, 7, 7});
auto* conv1_output_arg = helper.MakeIntermediate();
auto* conv2_output_arg = helper.MakeIntermediate();
auto* output_arg = helper.MakeOutput();
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<uint8_t>({1, 10, 7}, 0, 31);
auto* input2_arg = builder.MakeInput<uint8_t>({1, 12, 7, 7}, 0, 31);
auto* conv1_output_arg = builder.MakeIntermediate();
auto* conv2_output_arg = builder.MakeIntermediate();
auto* output_arg = builder.MakeOutput();
helper.AddQLinearConvNode<uint8_t>(input1_arg, .01f, 135,
{1, 10, 3}, .02f, 126,
conv1_output_arg, .37f, 131);
helper.AddQLinearConvNode<uint8_t>(input2_arg, .01f, 135,
{1, 12, 3, 3}, .02f, 126,
conv2_output_arg, .37f, 131);
auto* conv1_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {1, 10, 3});
builder.AddQLinearConvNode<uint8_t>(input1_arg, .01f, 135,
conv1_weight_arg, .02f, 126,
conv1_output_arg, .37f, 131);
auto* conv2_weight_arg = NhwcMakeInitializer<uint8_t>(builder, {1, 12, 3, 3});
builder.AddQLinearConvNode<uint8_t>(input2_arg, .01f, 135,
conv2_weight_arg, .02f, 126,
conv2_output_arg, .37f, 131);
// Broadcast add {1, 1, 5} to {1, 1, 5, 5}.
helper.AddQLinearBinaryNode("QLinearAdd",
conv1_output_arg, .37f, 131,
conv2_output_arg, .37f, 131,
output_arg, .39f, 126);
builder.AddQLinearBinaryNode("QLinearAdd",
conv1_output_arg, .37f, 131,
conv2_output_arg, .37f, 131,
output_arg, .39f, 126);
};
auto check_nhwc_graph = [&](InferenceSessionWrapper& session) {
@ -643,10 +435,13 @@ TEST(NhwcTransformerTests, ConvMixTensorRanks) {
// Generate a graph with QLinearAdd that broadcasts adds a 1D tensor to a
// 2D tensor and verify that the transformer handles the mixed tensor ranks.
NhwcTransformerTester(build_test_case, check_nhwc_graph);
TransformerTester(build_test_case,
check_nhwc_graph,
TransformerLevel::Level2,
TransformerLevel::Level3);
}
#endif // DISABLE_CONTRIB_OPS
#endif // DISABLE_CONTRIB_OPS
} // namespace test
} // namespace onnxruntime

237
onnxruntime/test/optimizer/qdq_transformer_test.cc Normal file
View file

@ -0,0 +1,237 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/graph/model.h"
#include "core/graph/onnx_protobuf.h"
#include "core/mlas/inc/mlas.h"
#include "core/session/environment.h"
#include "core/session/inference_session.h"
#include "test/compare_ortvalue.h"
#include "test/test_environment.h"
#include "test/framework/test_utils.h"
#include "test/util/include/inference_session_wrapper.h"
#include "gtest/gtest.h"
#include "graph_transform_test_builder.h"
namespace onnxruntime {
namespace test {
template <typename T>
typename std::enable_if<IsTypeQuantLinearCompatible<T>::value, NodeArg*>::type
AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale, T zp) {
auto* q_output = builder.MakeIntermediate();
auto* dq_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<T>(q_input, scale, zp, q_output);
builder.AddDequantizeLinearNode<T>(q_output, scale, zp, dq_output);
return dq_output;
}
template <typename T>
typename std::enable_if<IsTypeQuantLinearCompatible<T>::value, NodeArg*>::type
AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale) {
auto* q_output = builder.MakeIntermediate();
auto* dq_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode(q_input, scale, q_output);
builder.AddDequantizeLinearNode<T>(q_output, scale, dq_output);
return dq_output;
}
#ifndef DISABLE_CONTRIB_OPS
TEST(QDQTransformerTests, Conv) {
// TODO: enable fully use_default_zp tests after fixing inference bug in QuantizeLinear
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape, bool use_default_zp) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
auto* conv_output = builder.MakeIntermediate();
auto* weight = builder.MakeInitializer<uint8_t>(weights_shape, 0, 255);
auto* dq_w_output = builder.MakeIntermediate();
auto* dq_output = AddQDQNodePair<uint8_t>(builder, input_arg, .004f, 129);
if (use_default_zp) {
builder.AddDequantizeLinearNode<uint8_t>(weight, .003f, dq_w_output);
} else {
builder.AddDequantizeLinearNode<uint8_t>(weight, .003f, 118, dq_w_output);
}
builder.AddConvNode(dq_output, dq_w_output, conv_output);
builder.AddQuantizeLinearNode<uint8_t>(conv_output, .0039f, 135, output_arg);
};
auto check_conv_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["QLinearConv"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
TransformerTester(build_test_case, check_conv_graph, TransformerLevel::Level1, TransformerLevel::Level2);
};
// Test the basic case of a single 1D/2D/3D convolution.
test_case({1, 12, 37}, {32, 12, 5}, true);
test_case({1, 12, 37}, {32, 12, 5}, false);
test_case({1, 23, 13, 13}, {30, 23, 3, 3}, true);
test_case({1, 23, 13, 13}, {30, 23, 3, 3}, false);
test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}, true);
test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}, false);
}
TEST(QDQTransformerTests, ConvMaxPoolReshape) {
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
auto* weight = builder.MakeInitializer<uint8_t>(weights_shape, 0, 255);
// add QDQ + Conv
auto* dq_w_output = builder.MakeIntermediate();
auto* conv_output = builder.MakeIntermediate();
auto* dq_conv_output = AddQDQNodePair<uint8_t>(builder, input_arg, .004f, 129);
builder.AddDequantizeLinearNode<uint8_t>(weight, .003f, 118, dq_w_output);
builder.AddConvNode(dq_conv_output, dq_w_output, conv_output);
// add QDQ + MaxPool
auto* dq_maxpool_output = AddQDQNodePair<uint8_t>(builder, conv_output, .0039f, 135);
auto* maxpool_output = builder.MakeIntermediate();
Node& pool_node = builder.AddNode("MaxPool", {dq_maxpool_output}, {maxpool_output});
std::vector<int64_t> pads((weights_shape.size() - 2) * 2, 1);
pool_node.AddAttribute("pads", pads);
std::vector<int64_t> kernel_shape(weights_shape.size() - 2, 3);
pool_node.AddAttribute("kernel_shape", kernel_shape);
// add QDQ + Reshape
auto* dq_reshape_output = AddQDQNodePair<uint8_t>(builder, maxpool_output, .0039f, 135);
auto* reshape_shape = builder.Make1DInitializer<int64_t>({-1});
auto* reshape_output = builder.MakeIntermediate();
builder.AddNode("Reshape", {dq_reshape_output, reshape_shape}, {reshape_output});
// add Q
builder.AddQuantizeLinearNode<uint8_t>(reshape_output, .0039f, 135, output_arg);
};
auto check_mp_reshape_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["QLinearConv"], 1);
EXPECT_EQ(op_to_count["MaxPool"], 1);
EXPECT_EQ(op_to_count["Reshape"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
TransformerTester(build_test_case, check_mp_reshape_graph, TransformerLevel::Level1, TransformerLevel::Level2);
};
// Test the basic case of a single 1D/2D/3D convolution.
test_case({1, 12, 37}, {32, 12, 5});
test_case({1, 23, 13, 13}, {30, 23, 3, 3});
test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3});
}
TEST(QDQTransformerTests, Add) {
auto test_case = [&](const std::vector<int64_t>& input_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* input2_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
// add QDQ + Add
auto* add_output = builder.MakeIntermediate();
auto* dq_add_output1 = AddQDQNodePair<uint8_t>(builder, input1_arg, .004f, 129);
auto* dq_add_output2 = AddQDQNodePair<uint8_t>(builder, input2_arg, .004f, 129);
builder.AddNode("Add", {dq_add_output1, dq_add_output2}, {add_output});
// add Q
builder.AddQuantizeLinearNode<uint8_t>(add_output, .0039f, 135, output_arg);
};
auto check_add_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["com.microsoft.QLinearAdd"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
TransformerTester(build_test_case, check_add_graph, TransformerLevel::Level1, TransformerLevel::Level2);
};
// Test the basic case of a single 1D/2D/3D convolution.
test_case({1, 12, 37});
test_case({1, 23, 13, 13});
test_case({1, 22, 11, 13, 15});
}
TEST(QDQTransformerTests, Mul) {
auto test_case = [&](const std::vector<int64_t>& input_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* input2_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
// add QDQ + Mul
auto* mul_output = builder.MakeIntermediate();
auto* dq_mul_output1 = AddQDQNodePair<uint8_t>(builder, input1_arg, .004f, 129);
auto* dq_mul_output2 = AddQDQNodePair<uint8_t>(builder, input2_arg, .004f, 129);
builder.AddNode("Mul", {dq_mul_output1, dq_mul_output2}, {mul_output});
// add Q
builder.AddQuantizeLinearNode<uint8_t>(mul_output, .0039f, 135, output_arg);
};
auto check_mul_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["com.microsoft.QLinearMul"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
TransformerTester(build_test_case, check_mul_graph, TransformerLevel::Level1, TransformerLevel::Level2);
};
// Test the basic case of a single 1D/2D/3D convolution.
test_case({1, 12, 37});
test_case({1, 23, 13, 13});
test_case({1, 22, 11, 13, 15});
}
TEST(QDQTransformerTests, MatMul) {
auto test_case = [&](const std::vector<int64_t>& input1_shape, const std::vector<int64_t>& input2_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<float>(input1_shape, -1.f, 1.f);
auto* input2_arg = builder.MakeInput<float>(input2_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
// add QDQ + MatMul
auto* matmul_output = builder.MakeIntermediate();
auto* dq_matmul_output1 = AddQDQNodePair<uint8_t>(builder, input1_arg, .004f, 129);
auto* dq_matmul_output2 = AddQDQNodePair<uint8_t>(builder, input2_arg, .004f, 129);
builder.AddNode("MatMul", {dq_matmul_output1, dq_matmul_output2}, {matmul_output});
// add Q
builder.AddQuantizeLinearNode<uint8_t>(matmul_output, .0039f, 135, output_arg);
};
auto check_matmul_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["QLinearMatMul"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
TransformerTester(build_test_case, check_matmul_graph, TransformerLevel::Level1, TransformerLevel::Level2);
};
// Test the basic case of a single 1D/2D/3D convolution.
test_case({12, 37}, {37, 12});
test_case({23, 13, 13}, {13, 13});
test_case({22, 11, 13, 15}, {15, 13});
}
#endif // DISABLE_CONTRIB_OPS
} // namespace test
} // namespace onnxruntime