diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h
index fd13631f17342..b44b74467855e 100644
--- a/include/swift/Serialization/SerializedModuleLoader.h
+++ b/include/swift/Serialization/SerializedModuleLoader.h
@@ -163,8 +163,10 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   }
 
   /// Scan the given serialized module file to determine dependencies.
-  llvm::ErrorOr<ModuleDependencyInfo>
-  scanModuleFile(Twine modulePath, bool isFramework, bool isTestableImport);
+  llvm::ErrorOr<ModuleDependencyInfo> scanModuleFile(Twine modulePath,
+                                                     bool isFramework,
+                                                     bool isTestableImport,
+                                                     bool hasInterface);
 
   struct BinaryModuleImports {
     llvm::StringSet<> moduleImports;
diff --git a/lib/Serialization/ScanningLoaders.cpp b/lib/Serialization/ScanningLoaders.cpp
index 0bb53171a8c37..e7488c924403a 100644
--- a/lib/Serialization/ScanningLoaders.cpp
+++ b/lib/Serialization/ScanningLoaders.cpp
@@ -57,7 +57,8 @@ std::error_code SwiftModuleScanner::findModuleFilesInDirectory(
     if (fs.exists(ModPath)) {
       // The module file will be loaded directly.
       auto dependencies =
-          scanModuleFile(ModPath, IsFramework, isTestableDependencyLookup);
+          scanModuleFile(ModPath, IsFramework, isTestableDependencyLookup,
+                         /*hasInterface=*/false);
       if (dependencies) {
         this->dependencies = std::move(dependencies.get());
         return std::error_code();
@@ -159,8 +160,9 @@ SwiftModuleScanner::scanInterfaceFile(Twine moduleInterfacePath,
             !Ctx.SearchPathOpts.NoScannerModuleValidation) {
           assert(compiledCandidates.size() == 1 &&
                  "Should only have 1 candidate module");
-          auto BinaryDep = scanModuleFile(compiledCandidates[0], isFramework,
-                                          isTestableImport);
+          auto BinaryDep =
+              scanModuleFile(compiledCandidates[0], isFramework,
+                             isTestableImport, /*hasInterface=*/true);
           if (BinaryDep) {
             Result = *BinaryDep;
             return std::error_code();
diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp
index 2c607f409821a..0c326d98d3bfd 100644
--- a/lib/Serialization/SerializedModuleLoader.cpp
+++ b/lib/Serialization/SerializedModuleLoader.cpp
@@ -431,7 +431,8 @@ SerializedModuleLoaderBase::getImportsOfModule(
 
 llvm::ErrorOr<ModuleDependencyInfo>
 SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework,
-                                           bool isTestableImport) {
+                                           bool isTestableImport,
+                                           bool hasInterface) {
   const std::string moduleDocPath;
   const std::string sourceInfoPath;
 
@@ -455,7 +456,10 @@ SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework,
       return std::make_error_code(std::errc::no_such_file_or_directory);
     }
 
-    if (loadedModuleFile->isTestable() && !isTestableImport) {
+    // If the module file has interface file and not testable imported, don't
+    // import the testable module because it contains more interfaces than
+    // needed and can pull in more dependencies.
+    if (loadedModuleFile->isTestable() && !isTestableImport && hasInterface) {
       if (Ctx.LangOpts.EnableModuleLoadingRemarks)
         Ctx.Diags.diagnose(SourceLoc(), diag::skip_module_testable,
                            modulePath.str());
diff --git a/test/ScanDependencies/testable-dependencies.swift b/test/ScanDependencies/testable-dependencies.swift
index b16300be032e1..3230c38663d0b 100644
--- a/test/ScanDependencies/testable-dependencies.swift
+++ b/test/ScanDependencies/testable-dependencies.swift
@@ -56,12 +56,12 @@
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -enable-testing \
 // RUN:   -o %t/deps5.json -I %t/regular -swift-version 5 -Rmodule-loading
 
-/// Regular import a testable module with no interface, this is a dependency scanning error.
+/// Regular import a testable module with no interface, will try to import binary module but fail to look up the dependency.
 // RUN: rm %t/testable/A.swiftinterface
 // RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -module-name Test %t/main.swift \
 // RUN:   -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -enable-testing \
 // RUN:   -o %t/deps6.json -I %t/testable -swift-version 5 -Rmodule-loading 2>&1 | %FileCheck %s --check-prefix ERROR
-// ERROR: error: Unable to find module dependency: 'A'
+// ERROR: error: Unable to find module dependency: 'B'
 
 //--- main.swift
 import A