shards: populate RepoList.Stats.Repos (#605)

Previously this was only populated by the web package for serving a
specific endpoint. We now populate it in all places we do aggregation.

Test Plan: updated unit tests. Additionally loaded zoekt-webserver then
added to the index a repository which required 2 shards to index. The
repository count shown only increased by 1.

Author: keegancsmith

api.go

@@ -684,6 +684,9 @@ type IndexMetadata struct {
 // Statistics of a (collection of) repositories.
 type RepoStats struct {
 	// Repos is used for aggregrating the number of repositories.
+	//
+	// Note: This field is not populated on RepoListEntry.Stats (individual) but
+	// only for RepoList.Stats (aggregate).
 	Repos int
 
 	// Shards is the total number of search shards.

eval.go

@@ -731,6 +731,10 @@ func (d *indexData) List(ctx context.Context, q query.Q, opts *ListOptions) (rl
 
 	}
 
+	// Only one of these fields is populated and in all cases the size of that
+	// field is the number of Repos in this shard.
+	l.Stats.Repos = len(l.Repos) + len(l.Minimal) + len(l.ReposMap)
+
 	return &l, nil
 }
 

index_test.go

@@ -1711,6 +1711,7 @@ func TestListRepos(t *testing.T) {
 						},
 					}},
 					Stats: RepoStats{
+						Repos:        1,
 						Documents:    4,
 						ContentBytes: 68,
 						Shards:       1,
@@ -1772,6 +1773,7 @@ func TestListRepos(t *testing.T) {
 				},
 			},
 			Stats: RepoStats{
+				Repos:                      1,
 				Shards:                     1,
 				Documents:                  4,
 				IndexBytes:                 412,

shards/shards.go

@@ -987,6 +987,13 @@ func (ss *shardedSearcher) List(ctx context.Context, r query.Q, opts *zoekt.List
 		agg.Repos = append(agg.Repos, r)
 	}
 
+	// Only one of these fields is populated and in all cases the size of that
+	// field is the number of Repos.
+	//
+	// Note: we don't just add individual Stats.Repos since a repository can
+	// have multiple shards.
+	agg.Stats.Repos = len(uniq) + len(agg.Minimal) + len(agg.ReposMap)
+
 	if isAll && len(agg.Repos) > 0 {
 		reportListAllMetrics(agg.Repos)
 	}

shards/shards_test.go

@@ -472,6 +472,7 @@ func TestShardedSearcher_List(t *testing.T) {
 
 	aggStats := stats
 	aggStats.Add(&aggStats) // since both repos have the exact same stats, this works
+	aggStats.Repos = 2      // Add doesn't populate Repos, this is done in Shards	based on the response sizes.
 
 	for _, tc := range []struct {
 		name string

web/server.go

@@ -349,13 +349,7 @@ func (s *Server) fetchStats(ctx context.Context) (*zoekt.RepoStats, error) {
 		return nil, err
 	}
 
-	stats = &zoekt.RepoStats{}
-	names := map[string]struct{}{}
-	for _, r := range repos.Repos {
-		stats.Add(&r.Stats)
-		names[r.Repository.Name] = struct{}{}
-	}
-	stats.Repos = len(names)
+	stats = &repos.Stats
 
 	s.lastStatsMu.Lock()
 	s.lastStatsTS = time.Now()