Merge pull request git#185: unpack-trees: hash less

Creating the PR so I can generate installers and test the performance impact.

When updating the index using the cone-mode patterns for sparse-checkout, we should be able to immediately decide the new ce flags based on going out of the cone or hitting a recursive closure.

Modify `is_excluded_from_list()` to return a 2 for a recursive match, and use that value to immediately clear the skipworktree bit for everything in that directory. Similarly, if we get a 0, then keep the skipworktree bit on for everything in that directory. The value 1 means we saw that entry with a parent match, so should continue recursing.

Testing this on our target enlistment gave the following results for `git read-tree -mu HEAD`:

    Before: 49s
     After:  9s

dir.c

@@ -1221,7 +1221,7 @@ int is_excluded_from_list(const char *pathname,
 {
 	struct exclude *exclude;
 	struct strbuf parent_pathname = STRBUF_INIT;
-	int result = 0;
+	int result = IS_EXCLUDED_NOT_CONTAINS;
 	const char *slash_pos;
 
 	/*
@@ -1233,16 +1233,18 @@ int is_excluded_from_list(const char *pathname,
 	if (*dtype == DT_UNKNOWN)
 		*dtype = get_dtype(NULL, istate, pathname, pathlen);
 	if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
-		return 1;
+		return IS_EXCLUDED_CONTAINS_PARENT;
 
 	if (!el->use_cone_patterns) {
 		exclude = last_exclude_matching_from_list(pathname, pathlen, basename,
 								dtype, el, istate);
 
 		if (exclude)
-			return exclude->flags & EXC_FLAG_NEGATIVE ? 0 : 1;
+			return exclude->flags & EXC_FLAG_NEGATIVE
+			       ? IS_EXCLUDED_NOT_CONTAINS
+			       : IS_EXCLUDED_CONTAINS_PARENT;
 
-		return -1; /* undecided */
+		return IS_EXCLUDED_UNKNOWN; /* undecided */
 	}
 
 	strbuf_addch(&parent_pathname, '/');
@@ -1251,21 +1253,21 @@ int is_excluded_from_list(const char *pathname,
 
 	if (slash_pos == parent_pathname.buf) {
 		/* include every file in root */
-		result = 1;
+		result = IS_EXCLUDED_CONTAINS_PARENT;
 		goto done;
 	}
 
 	strbuf_setlen(&parent_pathname, slash_pos - parent_pathname.buf);
 
 	if (hashmap_contains_path(&el->parent_hashmap, &parent_pathname)) {
-		result = 1;
+		result = IS_EXCLUDED_CONTAINS_PARENT;
 		goto done;
 	}
 
 	while (parent_pathname.len) {
 		if (hashmap_contains_path(&el->recursive_hashmap,
 					  &parent_pathname)) {
-			result = -1;
+			result = IS_EXCLUDED_CONTAINS_RECURSIVE;
 			goto done;
 		}
 

dir.h

@@ -257,6 +257,10 @@ int read_directory(struct dir_struct *, struct index_state *istate,
 		   const char *path, int len,
 		   const struct pathspec *pathspec);
 
+#define IS_EXCLUDED_UNKNOWN -1
+#define IS_EXCLUDED_NOT_CONTAINS 0
+#define IS_EXCLUDED_CONTAINS_PARENT 1
+#define IS_EXCLUDED_CONTAINS_RECURSIVE 2
 int is_excluded_from_list(const char *pathname, int pathlen,
 			  const char *basename, int *dtype,
 			  struct exclude_list *el,

unpack-trees.c

@@ -1298,33 +1298,41 @@ static int clear_ce_flags_dir(struct index_state *istate,
 {
 	struct cache_entry **cache_end;
 	int dtype = DT_DIR;
-	int ret = is_excluded_from_list(prefix->buf, prefix->len,
-					basename, &dtype, el, istate);
+	int ret;
+	int orig_ret = is_excluded_from_list(prefix->buf, prefix->len,
+					     basename, &dtype, el, istate);
 	int rc;
 
 	strbuf_addch(prefix, '/');
 
 	/* If undecided, use matching result of parent dir in defval */
-	if (ret < 0)
+	if (orig_ret < 0)
 		ret = defval;
+	else
+		ret = orig_ret;
 
 	for (cache_end = cache; cache_end != cache + nr; cache_end++) {
 		struct cache_entry *ce = *cache_end;
 		if (strncmp(ce->name, prefix->buf, prefix->len))
 			break;
 	}
 
-	/*
-	 * TODO: check el, if there are no patterns that may conflict
-	 * with ret (iow, we know in advance the incl/excl
-	 * decision for the entire directory), clear flag here without
-	 * calling clear_ce_flags_1(). That function will call
-	 * the expensive is_excluded_from_list() on every entry.
-	 */
-	rc = clear_ce_flags_1(istate, cache, cache_end - cache,
-			      prefix,
-			      select_mask, clear_mask,
-			      el, ret);
+	if (el->use_cone_patterns && orig_ret == IS_EXCLUDED_CONTAINS_RECURSIVE) {
+		struct cache_entry **ce = cache;
+		rc = (cache_end - cache) / sizeof(struct cache_entry *);
+
+		while (ce < cache_end) {
+			(*ce)->ce_flags &= ~clear_mask;
+			ce++;
+		}
+	} else if (el->use_cone_patterns && orig_ret == IS_EXCLUDED_NOT_CONTAINS)
+		rc = (cache_end - cache) / sizeof(struct cache_entry *);
+	else
+		rc = clear_ce_flags_1(istate, cache, cache_end - cache,
+				      prefix,
+				      select_mask, clear_mask,
+				      el, ret);
+
 	strbuf_setlen(prefix, prefix->len - 1);
 	return rc;
 }