From 44cb4c4f74ba2d92f74b0ee53fab115952d1ed51 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Fri, 8 Mar 2019 10:30:20 -0800 Subject: [PATCH] Handle x86_64 -fPIC relocations correctly (#566) Thanks to cbecker for highlighting the problem in PR#565. That change used the GOT to access MlasMaskMoveAvx, but was incomplete: the GOT has a pointer to the global data, not the data directly. I updated the kernels to use a free register to load the address and then use that for the data. I ran through onnxruntime_mlas_test for AVX and FMA3 to verify the changes. --- .../core/mlas/lib/x86_64/LogisticKernelFma3.S | 5 +++-- onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S | 15 +++++++++------ .../core/mlas/lib/x86_64/SgemmKernelFma3.S | 9 ++++++--- .../core/mlas/lib/x86_64/SgemmKernelM1Avx.S | 5 +++-- .../mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S | 5 +++-- onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S | 5 +++-- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S index 243b355398..8b7b27dcbb 100644 --- a/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/LogisticKernelFma3.S @@ -72,7 +72,7 @@ Return Value: .globl C_UNDERSCORE(MlasLogisticKernelFma3) C_UNDERSCORE(MlasLogisticKernelFma3): - lea rax,C_UNDERSCORE(MlasLogisticConstants)[rip] + mov rax,C_UNDERSCORE(MlasLogisticConstants)@GOTPCREL[rip] vbroadcastss ymm4,LogisticConstants_LowerRange[rax] vbroadcastss ymm5,LogisticConstants_UpperRange[rax] vbroadcastss ymm6,LogisticConstants_alpha_9[rax] @@ -120,8 +120,9 @@ C_UNDERSCORE(MlasLogisticKernelFma3): add rdx,8 # correct for over-subtract above jz .LExitKernel mov DWORD PTR LogisticKernelFrame_CountN[rsp],edx + mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm2,DWORD PTR LogisticKernelFrame_CountN[rsp] - vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx] vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi] vmaxps ymm0,ymm4,ymm0 # clamp lower bound vminps ymm0,ymm5,ymm0 # clamp upper bound diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S index 63c6d5d2c8..0147f08edc 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelAvx.S @@ -374,9 +374,10 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x4Block: vmovd xmm0,r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] @@ -472,9 +473,10 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x2Block: vmovd xmm0,r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] @@ -538,9 +540,10 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Avx): .L\Mode\().OutputMasked8x1Block: vmovd xmm0,r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm0,xmm0,xmm0,0 - vpcmpgtd xmm1,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] - vpcmpgtd xmm0,xmm0,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd xmm1,xmm0,XMMWORD PTR [rbp+16] + vpcmpgtd xmm0,xmm0,XMMWORD PTR [rbp] vinsertf128 ymm0,ymm0,xmm1,1 .ifeqs "\Mode\()","Add" vmaskmovps ymm8,ymm0,YMMWORD PTR [rdx] diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S index a7382f8979..cfeceb6be3 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelFma3.S @@ -435,8 +435,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x6Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] @@ -549,8 +550,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x3Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vmaskmovps ymm6,ymm0,YMMWORD PTR [rdx+rax] @@ -651,8 +653,9 @@ C_UNDERSCORE(MlasSgemmKernel\Mode\()Fma3): .L\Mode\().OutputMasked8x1Block: mov DWORD PTR [rsp+SgemmKernelFrame_mask],r9d + mov rbp,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm0,DWORD PTR [rsp+SgemmKernelFrame_mask] - vpcmpgtd ymm0,ymm0,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd ymm0,ymm0,YMMWORD PTR [rbp] .ifeqs "\Mode\()","Add" vmaskmovps ymm4,ymm0,YMMWORD PTR [rdx] vfmadd213ps ymm5,ymm2,ymm4 diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S index 86bc82b230..28fca0e956 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1Avx.S @@ -80,9 +80,10 @@ C_UNDERSCORE(MlasSgemmKernelM1Avx): mov eax,r8d and eax,7 vmovd xmm7,eax + mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm7,xmm7,xmm7,0 - vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] - vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx] vinsertf128 ymm7,ymm7,xmm6,1 // diff --git a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S index 86bc9209fa..8d5ff17f90 100644 --- a/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S +++ b/onnxruntime/core/mlas/lib/x86_64/SgemmKernelM1TransposeBAvx.S @@ -79,9 +79,10 @@ C_UNDERSCORE(MlasSgemmKernelM1TransposeBAvx): mov eax,ecx and eax,7 vmovd xmm7,eax + mov rbx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vshufps xmm7,xmm7,xmm7,0 - vpcmpgtd xmm6,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip+16] - vpcmpgtd xmm7,xmm7,XMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd xmm6,xmm7,XMMWORD PTR [rbx+16] + vpcmpgtd xmm7,xmm7,XMMWORD PTR [rbx] vinsertf128 ymm7,ymm7,xmm6,1 // diff --git a/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S b/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S index dd5584648d..61bbef5c91 100644 --- a/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S +++ b/onnxruntime/core/mlas/lib/x86_64/TanhKernelFma3.S @@ -72,7 +72,7 @@ Return Value: .globl C_UNDERSCORE(MlasTanhKernelFma3) C_UNDERSCORE(MlasTanhKernelFma3): - lea rax,C_UNDERSCORE(MlasTanhConstants)[rip] + mov rax,C_UNDERSCORE(MlasTanhConstants)@GOTPCREL[rip] vbroadcastss ymm4,TanhConstants_LowerRange[rax] vbroadcastss ymm5,TanhConstants_UpperRange[rax] vbroadcastss ymm6,TanhConstants_alpha_13[rax] @@ -116,8 +116,9 @@ C_UNDERSCORE(MlasTanhKernelFma3): add rdx,8 # correct for over-subtract above jz .LExitKernel mov DWORD PTR TanhKernelFrame_CountN[rsp],edx + mov rcx,QWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)@GOTPCREL[rip] vbroadcastss ymm2,DWORD PTR TanhKernelFrame_CountN[rsp] - vpcmpgtd ymm2,ymm2,YMMWORD PTR C_UNDERSCORE(MlasMaskMoveAvx)[rip] + vpcmpgtd ymm2,ymm2,YMMWORD PTR [rcx] vmaskmovps ymm0,ymm2,YMMWORD PTR [rdi] vmaxps ymm0,ymm4,ymm0 # clamp lower bound vminps ymm0,ymm5,ymm0 # clamp upper bound