saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove @PCGOTREL x64 usage (#1707)

Browse source 
Avoid the need for @PCGOTREL relocations by annotating MLAS global data shared with assembly modules with attribute(visibility("hidden")).
This commit is contained in:
Tracy Sharpe 2019-08-28 11:27:16 -07:00 committed by GitHub
parent d9cdf4b4ed
commit 14eae293bf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 48 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onnxruntime/core/mlas/lib/amd64/QgemmU8U8KernelAvx2.asm
View file

@ -13,6 +13,8 @@
; This module implements the kernels for the quantized integer matrix/matrix
; multiply operation (QGEMM).
;
; This implementation uses AVX2 instructions.
;
;--
.xlist

2
onnxruntime/core/mlas/lib/erf.cpp
View file

@ -29,7 +29,7 @@ Abstract:
// Bundles the constants for use by kernels written in assembly.
//
extern "C" const struct {
MLAS_INTERNAL_DATA const struct {
float ErfUpperAbsRange;
float ErfSplitBoundary;
float ErfSMALL_P0;

2
onnxruntime/core/mlas/lib/logistic.cpp
View file

@ -26,7 +26,7 @@ Abstract:
// Bundles the floating point constants for use by kernels written in assembly.
//
extern "C" const struct {
MLAS_INTERNAL_DATA const struct {
float LowerRange;
float UpperRange;
float alpha_9;

16
onnxruntime/core/mlas/lib/mlasi.h
View file

@ -55,6 +55,18 @@ Abstract:
#define MLAS_FORCEINLINE __attribute__ ((always_inline)) inline
#endif
//
// Macro to tag globals as internal data shared with kernels written in
// assembly. These globals are marked with having hidden visibility to avoid
// needing to access the data through the global object table.
//
#if defined(_MSC_VER)
#define MLAS_INTERNAL_DATA extern "C"
#else
#define MLAS_INTERNAL_DATA extern "C" __attribute ((visibility("hidden")))
#endif
//
// Macro to suppress unreferenced parameter warnings.
//
@ -521,9 +533,7 @@ MlasGetMaximumThreadCount(
}
#endif
#if defined(MLAS_USE_WIN32_THREADPOOL)
return MlasPlatform.MaximumThreadCount;
#elif _OPENMP
#if defined(_OPENMP)
return (omp_get_num_threads() == 1) ? omp_get_max_threads() : 1;
#else
return 1;

2
onnxruntime/core/mlas/lib/sgemm.cpp
View file

@ -55,7 +55,7 @@ struct MLAS_SGEMM_WORK_BLOCK {
// Stores a vector to build a conditional load/store mask for vmaskmovps.
//
extern "C" MLAS_DECLSPEC_ALIGN(const uint32_t MlasMaskMoveAvx[8], 8 * sizeof(float)) = { 0, 1, 2, 3, 4, 5, 6, 7 };
MLAS_INTERNAL_DATA MLAS_DECLSPEC_ALIGN(const uint32_t MlasMaskMoveAvx[8], 8 * sizeof(float)) = { 0, 1, 2, 3, 4, 5, 6, 7 };
#endif

2
onnxruntime/core/mlas/lib/tanh.cpp
View file

@ -26,7 +26,7 @@ Abstract:
// Bundles the floating point constants for use by kernels written in assembly.
//
extern "C" const struct {
MLAS_INTERNAL_DATA const struct {
float LowerRange;
float UpperRange;
float alpha_13;

8
onnxruntime/core/mlas/lib/x86_64/ErfKernelFma3.S
View file

@ -26,6 +26,7 @@ Abstract:
//
// Structure layout for the erf constants block.
//
.equ ErfUpperAbsRange, 0
.equ ErfSplitBoundary, 4
.equ ErfSMALL_P0, 8
@ -68,7 +69,7 @@ Abstract:
.equ ErfBuffer1, 128
.equ ErfKernelFrame_CountN, 256
.equ ErfKernelFrame_ReturnAddress, 256+8
/*++
Routine Description:
@ -92,7 +93,7 @@ Return Value:
.globl C_UNDERSCORE(MlasErfKernelFma3)
C_UNDERSCORE(MlasErfKernelFma3):
sub rsp,ErfKernelFrame_ReturnAddress
mov rax,C_UNDERSCORE(MlasErfConstants)@GOTPCREL[rip]
lea rax,C_UNDERSCORE(MlasErfConstants)[rip]
sub rdx,8*4
jb .LErfProcessRemainingCount
@ -376,10 +377,9 @@ C_UNDERSCORE(MlasErfKernelFma3):
.LErfProcess1x8:
mov DWORD PTR ErfKernelFrame_CountN[rsp],edx
mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm3,DWORD PTR ErfKernelFrame_CountN[rsp]
vpcmpgtd ymm3,ymm3,YMMWORD PTR [rcx]
vpcmpgtd ymm3,ymm3,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vbroadcastss ymm15,ErfNegZero[rax]
vmaskmovps ymm0,ymm3,YMMWORD PTR [rdi] # original input vx0

5
onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S
View file

@ -72,7 +72,7 @@ Return Value:
.globl C_UNDERSCORE(MlasLogisticKernelFma3)
C_UNDERSCORE(MlasLogisticKernelFma3):
mov rax,C_UNDERSCORE(MlasLogisticConstants)@GOTPCREL[rip]
lea rax,C_UNDERSCORE(MlasLogisticConstants)[rip]
vbroadcastss ymm4,LogisticConstants_LowerRange[rax]
vbroadcastss ymm5,LogisticConstants_UpperRange[rax]
vbroadcastss ymm6,LogisticConstants_alpha_9[rax]
@ -120,9 +120,8 @@ C_UNDERSCORE(MlasLogisticKernelFma3):
add rdx,8 # correct for over-subtract above
jz .LExitKernel
mov DWORD PTR LogisticKernelFrame_CountN[rsp],edx
mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm2,DWORD PTR LogisticKernelFrame_CountN[rsp]
vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx]
vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi]
vmaxps ymm0,ymm4,ymm0 # clamp lower bound
vminps ymm0,ymm5,ymm0 # clamp upper bound

10
onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx2.S
View file

@ -6,13 +6,15 @@ Licensed under the MIT License.
Module Name:
QgemmU8U8KernelAvx2.S
QgemmU8U8KernelAvx2.s
Abstract:
This module implements the kernels for the quantized integer matrix/matrix
multiply operation (QGEMM).
This implementation uses AVX2 instructions.
--*/
#include "asmmacro.h"
@ -122,9 +124,8 @@ C_UNDERSCORE(MlasGemmU8U8CopyPackAAvx2):
inc eax
shr eax,1 # align unaligned count to pair count
mov DWORD PTR .LGemmU8U8CopyPackAFrame_mask[rsp],eax
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vpbroadcastd ymm9,DWORD PTR .LGemmU8U8CopyPackAFrame_mask[rsp]
vpcmpgtd ymm9,ymm9,YMMWORD PTR [rbp]
vpcmpgtd ymm9,ymm9,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
//
// Zero initialize the padded stack buffers.
@ -973,9 +974,8 @@ Implicit Arguments:
.LOutputMasked8xNBlock.\RowCount\():
mov DWORD PTR .LGemmU8U8KernelFrame_mask[rsp],r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vpbroadcastd ymm0,DWORD PTR .LGemmU8U8KernelFrame_mask[rsp]
vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp]
vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
test r14b,r14b # ZeroMode?
jnz .LSkipAccumulateOutputMasked8xNBlock.\RowCount\()
EmitIfCountGE \RowCount\(), 1, "vpmaskmovd ymm4,ymm0,YMMWORD PTR [rdx]"

2
onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512BW.S
View file

@ -6,7 +6,7 @@ Licensed under the MIT License.
Module Name:
QgemmU8U8KernelAvx512BW.S
QgemmU8U8KernelAvx512BW.s
Abstract:

2
onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Common.h
View file

@ -6,7 +6,7 @@ Licensed under the MIT License.
Module Name:
QgemmU8U8KernelAvx512Common.inc
QgemmU8U8KernelAvx512Common.h
Abstract:

2
onnxruntime/core/mlas/lib/x86_64/QgemmU8U8KernelAvx512Vnni.S
View file

@ -6,7 +6,7 @@ Licensed under the MIT License.
Module Name:
QgemmU8U8KernelAvx512Vnni.asm
QgemmU8U8KernelAvx512Vnni.s
Abstract:

15
onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S
View file

@ -374,10 +374,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx):
.L\Mode\().OutputMasked8x4Block:
vmovd xmm0,r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vshufps xmm0,xmm0,xmm0,0
vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp]
vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vinsertf128 ymm0,ymm0,xmm1,1
.ifeqs "\Mode\()","Add"
vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx]
@ -473,10 +472,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx):
.L\Mode\().OutputMasked8x2Block:
vmovd xmm0,r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vshufps xmm0,xmm0,xmm0,0
vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp]
vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vinsertf128 ymm0,ymm0,xmm1,1
.ifeqs "\Mode\()","Add"
vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx]
@ -540,10 +538,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx):
.L\Mode\().OutputMasked8x1Block:
vmovd xmm0,r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vshufps xmm0,xmm0,xmm0,0
vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp]
vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vinsertf128 ymm0,ymm0,xmm1,1
.ifeqs "\Mode\()","Add"
vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx]

9
onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S
View file

@ -435,9 +435,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3):
.L\Mode\().OutputMasked8x6Block:
mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask]
vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp]
vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
.ifeqs "\Mode\()","Add"
vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx]
vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax]
@ -550,9 +549,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3):
.L\Mode\().OutputMasked8x3Block:
mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask]
vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp]
vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
.ifeqs "\Mode\()","Add"
vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx]
vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax]
@ -653,9 +651,8 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3):
.L\Mode\().OutputMasked8x1Block:
mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d
mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask]
vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp]
vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
.ifeqs "\Mode\()","Add"
vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx]
vfmadd213ps ymm5,ymm2,ymm4

5
onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S
View file

@ -80,10 +80,9 @@ C_UNDERSCORE(MlasSgemmKernelM1Avx):
mov eax,r8d
and eax,7
vmovd xmm7,eax
mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vshufps xmm7,xmm7,xmm7,0
vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16]
vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx]
vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vinsertf128 ymm7,ymm7,xmm6,1
//

5
onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S
View file

@ -79,10 +79,9 @@ C_UNDERSCORE(MlasSgemmKernelM1TransposeBAvx):
mov eax,ecx
and eax,7
vmovd xmm7,eax
mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vshufps xmm7,xmm7,xmm7,0
vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16]
vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx]
vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16]
vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vinsertf128 ymm7,ymm7,xmm6,1
//

5
onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S
View file

@ -72,7 +72,7 @@ Return Value:
.globl C_UNDERSCORE(MlasTanhKernelFma3)
C_UNDERSCORE(MlasTanhKernelFma3):
mov rax,C_UNDERSCORE(MlasTanhConstants)@GOTPCREL[rip]
lea rax,C_UNDERSCORE(MlasTanhConstants)[rip]
vbroadcastss ymm4,TanhConstants_LowerRange[rax]
vbroadcastss ymm5,TanhConstants_UpperRange[rax]
vbroadcastss ymm6,TanhConstants_alpha_13[rax]
@ -116,9 +116,8 @@ C_UNDERSCORE(MlasTanhKernelFma3):
add rdx,8 # correct for over-subtract above
jz .LExitKernel
mov DWORD PTR TanhKernelFrame_CountN[rsp],edx
mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip]
vbroadcastss ymm2,DWORD PTR TanhKernelFrame_CountN[rsp]
vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx]
vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip]
vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi]
vmaxps ymm0,ymm4,ymm0 # clamp lower bound
vminps ymm0,ymm5,ymm0 # clamp upper bound