diff --git a/src/cargo/util/toml/targets.rs b/src/cargo/util/toml/targets.rs
index a2f4153a8c4..a7e30c61bce 100644
--- a/src/cargo/util/toml/targets.rs
+++ b/src/cargo/util/toml/targets.rs
@@ -624,10 +624,10 @@ fn infer_from_directory(directory: &Path) -> Vec<(String, PathBuf)> {
 }
 
 fn infer_any(entry: &DirEntry) -> Option<(String, PathBuf)> {
-    if entry.path().extension().and_then(|p| p.to_str()) == Some("rs") {
-        infer_file(entry)
-    } else if entry.file_type().map(|t| t.is_dir()).ok() == Some(true) {
+    if entry.file_type().map_or(false, |t| t.is_dir()) {
         infer_subdirectory(entry)
+    } else if entry.path().extension().and_then(|p| p.to_str()) == Some("rs") {
+        infer_file(entry)
     } else {
         None
     }
diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs
index 42238f2cff0..45eb1f2e0fe 100644
--- a/tests/testsuite/build.rs
+++ b/tests/testsuite/build.rs
@@ -5022,6 +5022,18 @@ fn inferred_benchmarks() {
     p.cargo("bench --bench=bar --bench=baz").run();
 }
 
+#[cargo_test]
+fn no_infer_dirs() {
+    let p = project()
+        .file("src/lib.rs", "fn main() {}")
+        .file("examples/dir.rs/dummy", "")
+        .file("benches/dir.rs/dummy", "")
+        .file("tests/dir.rs/dummy", "")
+        .build();
+
+    p.cargo("build --examples --benches --tests").run(); // should not fail with "is a directory"
+}
+
 #[cargo_test]
 fn target_edition() {
     let p = project()