diff --git a/tools/ci_build/amd_hipify.py b/tools/ci_build/amd_hipify.py
index 935b209229..7b1de841f7 100644
--- a/tools/ci_build/amd_hipify.py
+++ b/tools/ci_build/amd_hipify.py
@@ -12,6 +12,8 @@ contrib_ops_files = [
                     'bert/attention.h',
                     'bert/attention_impl.cu',
                     'bert/attention_impl.h',
+                    'bert/attention_transpose.cu',
+                    'bert/attention_past.cu',
                     'bert/embed_layer_norm.cc',
                     'bert/embed_layer_norm.h',
                     'bert/embed_layer_norm_impl.cu',