[ROCm] update hipify-perl location (#10102)

* [ROCm] update hipify-perl location

Depending on the ROCm version installed, hipify-perl might not always
live in the hard-coded path of /opt/rocm/bin. Use python 3.3's
shutil.which to locate the script.

* provide alternative locations for hipify-perl if not in PATH

* implement hipify-perl search as a function

This avoids running the logic during module import since all builds
import the amd_hipify module.

* fix flake8 errors

tools/ci_build/amd_hipify.py

@@ -2,7 +2,9 @@
 # Licensed under the MIT License.
 
 import concurrent.futures
+import functools
 import os
+import shutil
 import subprocess
 from logger import get_logger
 
@@ -173,7 +175,25 @@ training_ops_excluded_files = [
                     'cuda_training_kernels.h',
 ]
 
-HIPIFY_PERL = '/opt/rocm/bin/hipify-perl'
+
+@functools.lru_cache(maxsize=1)
+def get_hipify_path():
+    # prefer the hipify-perl in PATH
+    HIPIFY_PERL = shutil.which('hipify-perl')
+    # if not found, attempt hard-coded location 1
+    if HIPIFY_PERL is None:
+        print('hipify-perl not found, trying default location 1')
+        hipify_path = '/opt/rocm/hip/bin/hipify-perl'
+        HIPIFY_PERL = hipify_path if os.access(hipify_path, os.X_OK) else None
+    # if not found, attempt hard-coded location 2
+    if HIPIFY_PERL is None:
+        print('hipify-perl not found, trying default location 2')
+        hipify_path = '/opt/rocm/bin/hipify-perl'
+        HIPIFY_PERL = hipify_path if os.access(hipify_path, os.X_OK) else None
+    # fail
+    if HIPIFY_PERL is None:
+        raise RuntimeError('Could not locate hipify-perl script')
+    return HIPIFY_PERL
 
 
 def hipify(src_file_path, dst_file_path):
@@ -182,7 +202,7 @@ def hipify(src_file_path, dst_file_path):
     if not os.path.exists(dir_name):
         os.makedirs(dir_name, exist_ok=True)
     # Run hipify-perl first, capture output
-    s = subprocess.run([HIPIFY_PERL, src_file_path], stdout=subprocess.PIPE, universal_newlines=True).stdout
+    s = subprocess.run([get_hipify_path(), src_file_path], stdout=subprocess.PIPE, universal_newlines=True).stdout
 
     # Additional exact-match replacements.
     # Order matters for all of the following replacements, reglardless of appearing in logical sections.
@@ -323,6 +343,8 @@ def list_files(prefix, path):
 
 
 def amd_hipify(config_build_dir):
+    # determine hipify script path now to avoid doing so concurrently in the thread pool
+    print('Using %s' % get_hipify_path())
     with concurrent.futures.ThreadPoolExecutor() as executor:
         cuda_path = os.path.join(contrib_ops_path, 'cuda')
         rocm_path = os.path.join(config_build_dir, 'amdgpu', contrib_ops_path, 'rocm')