diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index b3c9d5908654f..4a35da6140b2a 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -486,10 +486,16 @@ void RocmInstallationDetector::detectHIPRuntime() {
       return newpath;
     };
     // If HIP version file can be found and parsed, use HIP version from there.
-    for (const auto &VersionFilePath :
-         {Append(SharePath, "hip", "version"),
-          Append(ParentSharePath, "hip", "version"),
-          Append(BinPath, ".hipVersion")}) {
+    std::vector<SmallString<0>> VersionFilePaths = {
+        Append(SharePath, "hip", "version"),
+        InstallPath != D.SysRoot + "/usr/local"
+            ? Append(ParentSharePath, "hip", "version")
+            : SmallString<0>(),
+        Append(BinPath, ".hipVersion")};
+
+    for (const auto &VersionFilePath : VersionFilePaths) {
+      if (VersionFilePath.empty())
+        continue;
       llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> VersionFile =
           FS.getBufferForFile(VersionFilePath);
       if (!VersionFile)
diff --git a/clang/test/Driver/rocm-detect.hip b/clang/test/Driver/rocm-detect.hip
index 3644f215a345b..0db994af556f3 100644
--- a/clang/test/Driver/rocm-detect.hip
+++ b/clang/test/Driver/rocm-detect.hip
@@ -77,8 +77,18 @@
 // RUN:   --hip-path=%t/myhip --print-rocm-search-dirs %s 2>&1 \
 // RUN:   | FileCheck -check-prefixes=ROCM-ENV,HIP-PATH %s
 
+// Test detecting /usr directory.
+// RUN: rm -rf %t/*
+// RUN: cp -r %S/Inputs/rocm %t/usr
+// RUN: mkdir -p %t/usr/share/hip
+// RUN: mv %t/usr/bin/.hipVersion %t/usr/share/hip/version
+// RUN: mkdir -p %t/usr/local
+// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=gfx1010 --sysroot=%t \
+// RUN:   --print-rocm-search-dirs --hip-link %s 2>&1 \
+// RUN:   | FileCheck -check-prefixes=USR %s
+
 // Test detecting latest /opt/rocm-{release} directory.
-// RUN: rm -rf %t/opt
+// RUN: rm -rf %t/*
 // RUN: mkdir -p %t/opt
 // RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.9.0-1234
 // RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.10.0
@@ -130,6 +140,11 @@
 // ROCM-PATH: "-idirafter" "[[ROCM_PATH]]/include"
 // ROCM-PATH: "-L[[ROCM_PATH]]/lib" {{.*}}"-lamdhip64"
 
+// USR: ROCm installation search path: [[ROCM_PATH:.*/usr$]]
+// USR: "-mlink-builtin-bitcode" "[[ROCM_PATH]]/amdgcn/bitcode/oclc_isa_version_1010.bc"
+// USR: "-idirafter" "[[ROCM_PATH]]/include"
+// USR: "-L[[ROCM_PATH]]/lib" {{.*}}"-lamdhip64"
+
 // ROCM-REL: ROCm installation search path: {{.*}}/opt/rocm
 // ROCM-REL: ROCm installation search path: {{.*}}/opt/rocm-3.10.0