diff --git a/c10/cuda/test/CMakeLists.txt b/c10/cuda/test/CMakeLists.txt
index 05336496a47..10027062533 100644
--- a/c10/cuda/test/CMakeLists.txt
+++ b/c10/cuda/test/CMakeLists.txt
@@ -14,7 +14,13 @@ if(BUILD_TEST)
   foreach(test_src ${C10_CUDA_ALL_TEST_FILES})
     get_filename_component(test_file_name ${test_src} NAME_WE)
     set(test_name "c10_cuda_${test_file_name}")
-    add_executable(${test_name} "${test_src}")
+    if(WIN32 AND test_src MATCHES "^.*\.hip$")
+      set_source_files_properties(${test_src} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
+      hip_add_executable(${test_name} "${test_src}")
+      set_target_properties(${test_name} PROPERTIES LINKER_LANGUAGE CXX HIP_ARCHITECTURES ${PYTORCH_ROCM_ARCH})
+    else()
+      add_executable(${test_name} "${test_src}")
+    endif()
     target_link_libraries(${test_name} ${C10_CUDA_LIB} ${C10_LIB} gtest_main)
     add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
     if(INSTALL_TEST)
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index 114ce44adb9..29fb2f53dd3 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -1013,7 +1013,23 @@ if(USE_ROCM)
       caffe2_update_option(USE_SYSTEM_NCCL ON)
     endif()
 
-    list(APPEND HIP_CXX_FLAGS -fPIC)
+    if(WIN32)
+      if(${CAFFE2_USE_MSVC_STATIC_RUNTIME})
+        if(CMAKE_BUILD_TYPE MATCHES Debug)
+          list(APPEND HIP_CXX_FLAGS -fms-runtime-lib=static_dbg)
+        else()
+          list(APPEND HIP_CXX_FLAGS -fms-runtime-lib=static)
+        endif()
+      else()
+        if(CMAKE_BUILD_TYPE MATCHES Debug)
+          list(APPEND HIP_CXX_FLAGS -fms-runtime-lib=dll_dbg)
+        else()
+          list(APPEND HIP_CXX_FLAGS -fms-runtime-lib=dll)
+        endif()
+      endif()
+    else()
+      list(APPEND HIP_CXX_FLAGS -fPIC)
+    endif()
     list(APPEND HIP_CXX_FLAGS -D__HIP_PLATFORM_AMD__=1)
     list(APPEND HIP_CXX_FLAGS -DCUDA_HAS_FP16=1)
     list(APPEND HIP_CXX_FLAGS -DUSE_ROCM)
diff --git a/cmake/public/LoadHIP.cmake b/cmake/public/LoadHIP.cmake
index 273dcbe3e72..9dbb9ebb2ee 100644
--- a/cmake/public/LoadHIP.cmake
+++ b/cmake/public/LoadHIP.cmake
@@ -192,35 +192,34 @@ if(HIP_FOUND)
       #message("HIP is NOT using new type enums: ${hip_compile_output}")
       message("HIP is NOT using new type enums")
     endif()
+
+    if(ROCM_VERSION_DEV VERSION_GREATER_EQUAL "5.7.0")
+      # check whether hipblaslt provides HIPBLASLT_MATMUL_DESC_A_SCALE_POINTER_VEC_EXT
+      set(file "${PROJECT_BINARY_DIR}/hipblaslt_test_vec_ext.cc")
+      file(WRITE ${file} ""
+        "#define LEGACY_HIPBLAS_DIRECT\n"
+        "#include <hipblaslt/hipblaslt.h>\n"
+        "int main() {\n"
+        "    hipblasLtMatmulDescAttributes_t attr = HIPBLASLT_MATMUL_DESC_A_SCALE_POINTER_VEC_EXT;\n"
+        "    return 0;\n"
+        "}\n"
+        )
+      try_compile(hipblaslt_compile_result_vec_ext ${PROJECT_RANDOM_BINARY_DIR} ${file}
+        CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${ROCM_INCLUDE_DIRS}"
+        COMPILE_DEFINITIONS -D__HIP_PLATFORM_AMD__ -D__HIP_PLATFORM_HCC__
+        OUTPUT_VARIABLE hipblaslt_compile_output)
+      if(hipblaslt_compile_result_vec_ext)
+        set(HIPBLASLT_VEC_EXT ON)
+        #message("hipblaslt is using scale pointer vec ext: ${hipblaslt_compile_output}")
+        message("hipblaslt is using scale pointer vec ext")
+      else()
+        set(HIPBLASLT_VEC_EXT OFF)
+        message("hipblaslt is NOT using scale pointer vec ext: ${hipblaslt_compile_output}")
+        #message("hipblaslt is NOT using scale pointer vec ext")
+      endif()
+    endif()
   else() # Win32
     # With HIP-SDK 6.2, HIP declares new enum types on Windows
     set(HIP_NEW_TYPE_ENUMS ON)
   endif()
-
-  if(ROCM_VERSION_DEV VERSION_GREATER_EQUAL "5.7.0")
-    # check whether hipblaslt provides HIPBLASLT_MATMUL_DESC_A_SCALE_POINTER_VEC_EXT
-    set(file "${PROJECT_BINARY_DIR}/hipblaslt_test_vec_ext.cc")
-    file(WRITE ${file} ""
-      "#define LEGACY_HIPBLAS_DIRECT\n"
-      "#include <hipblaslt/hipblaslt.h>\n"
-      "int main() {\n"
-      "    hipblasLtMatmulDescAttributes_t attr = HIPBLASLT_MATMUL_DESC_A_SCALE_POINTER_VEC_EXT;\n"
-      "    return 0;\n"
-      "}\n"
-      )
-    try_compile(hipblaslt_compile_result_vec_ext ${PROJECT_RANDOM_BINARY_DIR} ${file}
-      CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${ROCM_INCLUDE_DIRS}"
-      COMPILE_DEFINITIONS -D__HIP_PLATFORM_AMD__ -D__HIP_PLATFORM_HCC__
-      OUTPUT_VARIABLE hipblaslt_compile_output)
-    if(hipblaslt_compile_result_vec_ext)
-      set(HIPBLASLT_VEC_EXT ON)
-      #message("hipblaslt is using scale pointer vec ext: ${hipblaslt_compile_output}")
-      message("hipblaslt is using scale pointer vec ext")
-    else()
-      set(HIPBLASLT_VEC_EXT OFF)
-      message("hipblaslt is NOT using scale pointer vec ext: ${hipblaslt_compile_output}")
-      #message("hipblaslt is NOT using scale pointer vec ext")
-    endif()
-  endif()
-
 endif()