Skip to content

Commit

Permalink
feat(serving): return estimates for all ref edge subkinds (#5784)
Browse files Browse the repository at this point in the history 
Instead of returning a total count of ref edges, return a map containing the
count of each ref subkind. With ref/writes, clients want to have more visibility
into the counts of different ref kinds.
  • Loading branch information
@salguarnieri
salguarnieri committed Aug 11, 2023
1 parent a92f67c commit a9402c7
Show file tree
Hide file tree
Showing 4 changed files with 283 additions and 180 deletions.
44 changes: 41 additions & 3 deletions kythe/go/serving/xrefs/xrefs.go
View file
Expand Up @@ -655,8 +655,10 @@ func (c xrefCategory) AddCount(reply *xpb.CrossReferencesReply, idx *srvpb.Paged
}
case xrefCategoryRef:
if pageSet.Contains(idx) {
reply.Total.RefEdgeToCount[strings.TrimPrefix(idx.Kind, "%")] += int64(idx.Count)
reply.Total.References += int64(idx.Count)
} else {
reply.Filtered.RefEdgeToCount[strings.TrimPrefix(idx.Kind, "%")] += int64(idx.Count)
reply.Filtered.References += int64(idx.Count)
}
case xrefCategoryRelated:
Expand Down Expand Up @@ -758,11 +760,17 @@ func (t *Table) CrossReferences(ctx context.Context, req *xpb.CrossReferencesReq
CrossReferences: make(map[string]*xpb.CrossReferencesReply_CrossReferenceSet, len(req.Ticket)),
Nodes: make(map[string]*cpb.NodeInfo, len(req.Ticket)),

Total: &xpb.CrossReferencesReply_Total{},
Total: &xpb.CrossReferencesReply_Total{
RefEdgeToCount: make(map[string]int64),
},
Filtered: &xpb.CrossReferencesReply_Total{
RefEdgeToCount: make(map[string]int64),
RelatedNodesByRelation: make(map[string]int64),
},
}
// Before we return reply, remove all RefEdgeToCount map entries that point to a 0 count.
defer cleanupRefEdgeToCount(reply)

if len(req.Filter) > 0 {
reply.Total.RelatedNodesByRelation = make(map[string]int64)
}
Expand Down Expand Up @@ -908,8 +916,11 @@ readLoop:
}
case xrefs.IsRefKind(req.ReferenceKind, grp.Kind):
filtered := filter.FilterGroup(grp)
reply.Total.RefEdgeToCount[strings.TrimPrefix(grp.Kind, "%")] += int64(len(grp.Anchor))
reply.Total.References += int64(len(grp.Anchor))
reply.Total.RefEdgeToCount[strings.TrimPrefix(grp.Kind, "%")] += int64(countRefs(grp.GetScopedReference()))
reply.Total.References += int64(countRefs(grp.GetScopedReference()))
reply.Filtered.RefEdgeToCount[strings.TrimPrefix(grp.Kind, "%")] += int64(filtered)
reply.Filtered.References += int64(filtered)
if wantMoreCrossRefs {
stats.addAnchors(&crs.Reference, grp)
Expand Down Expand Up @@ -1048,7 +1059,9 @@ readLoop:
if err != nil {
return nil, fmt.Errorf("internal error: error retrieving cross-references page %v: %v", idx.PageKey, err)
}
reply.Total.References -= int64(filtered) // update counts to reflect filtering
reply.Total.RefEdgeToCount[strings.TrimPrefix(idx.Kind, "%")] -= int64(filtered) // update counts to reflect filtering
reply.Total.References -= int64(filtered) // update counts to reflect filtering
reply.Filtered.RefEdgeToCount[strings.TrimPrefix(idx.Kind, "%")] += int64(filtered)
reply.Filtered.References += int64(filtered)
stats.addAnchors(&crs.Reference, p.Group)
}
Expand Down Expand Up @@ -1231,6 +1244,22 @@ readLoop:
return reply, nil
}

// cleanupRefEdgeToCount removes all the keys from r.Total.RefEdgeToCount and
// r.Filtered.RefEdgeToCount that have a value of 0.
func cleanupRefEdgeToCount(r *xpb.CrossReferencesReply) {
for k, v := range r.Total.RefEdgeToCount {
if v == 0 {
delete(r.Total.RefEdgeToCount, k)
}
}
for k, v := range r.Filtered.RefEdgeToCount {
if v == 0 {
delete(r.Filtered.RefEdgeToCount, k)
}
}

}

func addMergeNode(mergeMap map[string]string, allTickets []string, rootNode, mergeNode string) []string {
if _, ok := mergeMap[mergeNode]; ok {
return allTickets
Expand Down Expand Up @@ -1261,11 +1290,20 @@ func nodeKind(n *srvpb.Node) string {
}

func sumTotalCrossRefs(ts *xpb.CrossReferencesReply_Total) int {
var refs int
for _, cnt := range ts.RefEdgeToCount {
refs += int(cnt)
}
var relatedNodes int
for _, cnt := range ts.RelatedNodesByRelation {
relatedNodes += int(cnt)
}
return int(ts.Callers) + int(ts.Definitions) + int(ts.Declarations) + int(ts.References) + int(ts.Documentation) + relatedNodes
return int(ts.Callers) +
int(ts.Definitions) +
int(ts.Declarations) +
refs +
int(ts.Documentation) +
relatedNodes
}

type refOptions struct {
Expand Down
42 changes: 39 additions & 3 deletions kythe/go/serving/xrefs/xrefs_test.go
View file
Expand Up @@ -1331,10 +1331,14 @@ func TestCrossReferences(t *testing.T) {
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
Definitions: 1,
References: 2,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 2,
},
}, reply.Total); err != nil {
t.Error(err)
}
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{}, reply.Filtered); err != nil {
wantFiltered := &xpb.CrossReferencesReply_Total{}
if err := testutil.DeepEqual(wantFiltered, reply.Filtered); err != nil {
t.Error(err)
}

Expand Down Expand Up @@ -1494,10 +1498,14 @@ func TestCrossReferencesScoped(t *testing.T) {
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
Definitions: 1,
References: 2,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 2,
},
}, reply.Total); err != nil {
t.Error(err)
}
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{}, reply.Filtered); err != nil {
wantFiltered := &xpb.CrossReferencesReply_Total{}
if err := testutil.DeepEqual(wantFiltered, reply.Filtered); err != nil {
t.Error(err)
}

Expand Down Expand Up @@ -1788,10 +1796,14 @@ func TestCrossReferencesReadAhead(t *testing.T) {
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
Definitions: 1,
References: 2,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 2,
},
}, reply.Total); err != nil {
t.Error(err)
}
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{}, reply.Filtered); err != nil {
wantFiltered := &xpb.CrossReferencesReply_Total{}
if err := testutil.DeepEqual(wantFiltered, reply.Filtered); err != nil {
t.Error(err)
}

Expand Down Expand Up @@ -2188,11 +2200,17 @@ filter: {
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
Definitions: 1,
References: 1,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 1,
},
}, reply.Total); err != nil {
t.Error(err)
}
if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 1,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 1,
},
}, reply.Filtered); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2477,6 +2495,9 @@ func TestCrossReferencesIndirection(t *testing.T) {

if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 1,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 1,
},
}, reply.Total); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2543,6 +2564,9 @@ func TestCrossReferencesIndirection(t *testing.T) {

if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 3,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 3,
},
}, reply.Total); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2612,6 +2636,9 @@ func TestCrossReferencesIndirection(t *testing.T) {

if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 3,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 3,
},
}, reply.Total); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2664,6 +2691,9 @@ func TestCrossReferencesIndirection(t *testing.T) {

if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 2,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 2,
},
}, reply.Total); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2742,6 +2772,9 @@ func TestCrossReferencesIndirection(t *testing.T) {

if err := testutil.DeepEqual(&xpb.CrossReferencesReply_Total{
References: 4,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 4,
},
}, reply.Total); err != nil {
t.Error(err)
}
Expand Down Expand Up @@ -2884,6 +2917,9 @@ func TestCrossReferencesRevisions(t *testing.T) {
expected := &xpb.CrossReferencesReply{
Total: &xpb.CrossReferencesReply_Total{
References: 1,
RefEdgeToCount: map[string]int64{
"/kythe/edge/ref": 1,
},
},
Filtered: &xpb.CrossReferencesReply_Total{},
CrossReferences: map[string]*xpb.CrossReferencesReply_CrossReferenceSet{
Expand Down
8 changes: 7 additions & 1 deletion kythe/proto/xref.proto
View file
Expand Up @@ -566,10 +566,16 @@ message CrossReferencesReply {
message Total {
int64 definitions = 1;
int64 declarations = 2;
int64 references = 3;
int64 references = 3 [deprecated = true];
int64 documentation = 4;
int64 callers = 5;

// Map from ref edge type to count.
//
// Example
// /kythe/edge/ref -> 20,
// /kythe/edge/ref/writes -> 50
map<string, int64> ref_edge_to_count = 7;
map<string, int64> related_nodes_by_relation = 6;
}
// Total number of cross-references on all pages matching requested kinds,
Expand Down

0 comments on commit a9402c7

Please sign in to comment.