setup_current_filesystem: fail if name_max is 0

Add error handling to the USE_READDIR_R code paths that set name_max
from struct statfs or statvfs; if the determined name_max == 0
then return an error.

Avoids a crash in tree_dir_next_posix() when the calculation of
dirent_size from name_max is too small for the memory allocated
for struct dirent.

This may fix Github issue #1149
This may fix NetBSD PR https://gnats.netbsd.org/56080

libarchive/archive_read_disk_posix.c

@@ -1670,6 +1670,11 @@ setup_current_filesystem(struct archive_read_disk *a)
 	else
 		t->current_filesystem->name_max = nm;
 #endif
+	if (t->current_filesystem->name_max == 0) {
+		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+		    "Cannot determine name_max");
+		return (ARCHIVE_FAILED);
+	}
 #endif /* USE_READDIR_R */
 	return (ARCHIVE_OK);
 }
@@ -1865,6 +1870,11 @@ setup_current_filesystem(struct archive_read_disk *a)
 #else
 	t->current_filesystem->name_max = sfs.f_namelen;
 #endif
+	if (t->current_filesystem->name_max == 0) {
+		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+		    "Cannot determine name_max");
+		return (ARCHIVE_FAILED);
+	}
 #endif
 	return (ARCHIVE_OK);
 }
@@ -1946,6 +1956,11 @@ setup_current_filesystem(struct archive_read_disk *a)
 #if defined(USE_READDIR_R)
 	/* Set maximum filename length. */
 	t->current_filesystem->name_max = svfs.f_namemax;
+	if (t->current_filesystem->name_max == 0) {
+		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+		    "Cannot determine name_max");
+		return (ARCHIVE_FAILED);
+	}
 #endif
 	return (ARCHIVE_OK);
 }
@@ -2000,6 +2015,11 @@ setup_current_filesystem(struct archive_read_disk *a)
 	else
 		t->current_filesystem->name_max = nm;
 #  endif /* _PC_NAME_MAX */
+	if (t->current_filesystem->name_max == 0) {
+		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+		    "Cannot determine name_max");
+		return (ARCHIVE_FAILED);
+	}
 #endif /* USE_READDIR_R */
 	return (ARCHIVE_OK);
 }