Skip to content

Commit

Permalink
feat(post-processing): add diagnostics to file decorations (#3277)
Browse files Browse the repository at this point in the history
  • Loading branch information
@schroederc
schroederc committed Nov 20, 2018
1 parent 1ea13da commit 0cd5dfc
Show file tree
Hide file tree
Showing 8 changed files with 324 additions and 58 deletions.
141 changes: 122 additions & 19 deletions kythe/go/serving/pipeline/beam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,12 @@ func init() {
beam.RegisterFunction(completeDocument)
beam.RegisterFunction(constructCaller)
beam.RegisterFunction(defToDecorPiece)
beam.RegisterFunction(diagToDecor)
beam.RegisterFunction(edgeTargets)
beam.RegisterFunction(edgeToCrossRefRelation)
beam.RegisterFunction(emitRelatedDefs)
beam.RegisterFunction(fileToDecorPiece)
beam.RegisterFunction(fileToTags)
beam.RegisterFunction(filterAnchorNodes)
beam.RegisterFunction(groupCrossRefs)
beam.RegisterFunction(groupEdges)
Expand All @@ -65,22 +67,26 @@ func init() {
beam.RegisterFunction(moveSourceToKey)
beam.RegisterFunction(nodeToChildren)
beam.RegisterFunction(nodeToDecorPiece)
beam.RegisterFunction(nodeToDiagnostic)
beam.RegisterFunction(nodeToDocs)
beam.RegisterFunction(nodeToEdges)
beam.RegisterFunction(nodeToReverseEdges)
beam.RegisterFunction(parseMarkedSource)
beam.RegisterFunction(refToCallsite)
beam.RegisterFunction(refToCrossRef)
beam.RegisterFunction(refToDecorPiece)
beam.RegisterFunction(refToTag)
beam.RegisterFunction(reverseEdge)
beam.RegisterFunction(splitEdge)
beam.RegisterFunction(targetToFile)
beam.RegisterFunction(toDefinition)
beam.RegisterFunction(toEnclosingFile)
beam.RegisterFunction(toFiles)
beam.RegisterFunction(toRefs)

beam.RegisterType(reflect.TypeOf((*combineDecorPieces)(nil)).Elem())
beam.RegisterType(reflect.TypeOf((*ticketKey)(nil)).Elem())

beam.RegisterType(reflect.TypeOf((*cpb.Diagnostic)(nil)).Elem())
beam.RegisterType(reflect.TypeOf((*cpb.MarkedSource)(nil)).Elem())
beam.RegisterType(reflect.TypeOf((*ppb.DecorationPiece)(nil)).Elem())
beam.RegisterType(reflect.TypeOf((*ppb.Reference)(nil)).Elem())
Expand Down Expand Up @@ -313,19 +319,99 @@ func keyRef(r *ppb.Reference) (*spb.VName, *ppb.Reference) {
}

func (k *KytheBeam) decorationPieces(s beam.Scope) beam.PCollection {
targets := beam.ParDo(s, toEnclosingFile, k.References())
decor := beam.ParDo(s, refToDecorPiece, k.References())

targets := beam.ParDo(s, targetToFile, decor)
bareNodes := beam.ParDo(s, &nodes.Filter{IncludeEdges: []string{}}, k.nodes)

decor := beam.ParDo(s, refToDecorPiece, k.References())
files := beam.ParDo(s, fileToDecorPiece, k.getFiles())
nodes := beam.ParDo(s, nodeToDecorPiece,
targetNodes := beam.ParDo(s, nodeToDecorPiece,
beam.CoGroupByKey(s, beam.ParDo(s, moveSourceToKey, bareNodes), targets))
defs := beam.ParDo(s, defToDecorPiece,
beam.CoGroupByKey(s, k.directDefinitions(), targets))
// TODO(schroederc): overrides
// TODO(schroederc): diagnostics
decorDiagnostics := k.diagnostics()

return beam.Flatten(s, decor, files, nodes, defs)
return beam.Flatten(s, decor, files, targetNodes, defs, decorDiagnostics)
}

func (k *KytheBeam) diagnostics() beam.PCollection {
s := k.s.Scope("Diagnostics")
diagnostics := beam.Seq(s, k.Nodes(), &nodes.Filter{
FilterByKind: []string{kinds.Diagnostic},
IncludeFacts: []string{facts.Message, facts.Details, facts.ContextURL},
}, nodeToDiagnostic)
refTags := beam.ParDo(s, refToTag, k.References())
fileTags := beam.Seq(s, k.Nodes(), &nodes.Filter{
FilterByKind: []string{kinds.File},
IncludeFacts: []string{},
IncludeEdges: []string{edges.Tagged},
}, fileToTags)
return beam.ParDo(s, diagToDecor, beam.CoGroupByKey(s, diagnostics, refTags, fileTags))
}

func fileToTags(n *scpb.Node, emit func(*spb.VName, *spb.VName)) {
for _, e := range n.Edge {
emit(e.Target, n.Source)
}
}

func diagToDecor(src *spb.VName, diagStream func(**cpb.Diagnostic) bool, refTagStream func(**srvpb.ExpandedAnchor) bool, fileTagStream func(**spb.VName) bool, emit func(*spb.VName, *ppb.DecorationPiece)) error {
var d *cpb.Diagnostic
if !diagStream(&d) {
return nil
}

var ref *srvpb.ExpandedAnchor
for refTagStream(&ref) {
uri, err := kytheuri.Parse(ref.Ticket)
if err != nil {
return err
}
file := &spb.VName{
Corpus: uri.Corpus,
Root: uri.Root,
Path: uri.Path,
}
diagWithSpan := *d
diagWithSpan.Span = ref.Span
emit(file, &ppb.DecorationPiece{
Piece: &ppb.DecorationPiece_Diagnostic{
Diagnostic: &diagWithSpan,
},
})
}

var file *spb.VName
for fileTagStream(&file) {
emit(file, &ppb.DecorationPiece{
Piece: &ppb.DecorationPiece_Diagnostic{Diagnostic: d},
})
}

return nil
}

func refToTag(r *ppb.Reference, emit func(*spb.VName, *srvpb.ExpandedAnchor)) {
if r.GetKytheKind() != scpb.EdgeKind_TAGGED {
return
}
emit(r.Source, r.Anchor)
}

func nodeToDiagnostic(n *scpb.Node) (*spb.VName, *cpb.Diagnostic) {
d := &cpb.Diagnostic{}
for _, f := range n.Fact {
switch f.GetKytheName() {
case scpb.FactName_MESSAGE:
d.Message = string(f.Value)
case scpb.FactName_DETAILS:
d.Details = string(f.Value)
case scpb.FactName_CONTEXT_URL:
d.ContextUrl = string(f.Value)
}
}
return n.Source, d
}

// SplitDecorations returns a columnar Kythe file decorations table derived from
Expand All @@ -351,13 +437,8 @@ func (t *ticketKey) ProcessElement(key *spb.VName, val beam.T) (string, beam.T)
return t.Prefix + kytheuri.ToString(key), val
}

func toEnclosingFile(r *ppb.Reference) (*spb.VName, *spb.VName, error) {
anchor, err := kytheuri.ToVName(r.Anchor.Ticket)
if err != nil {
return nil, nil, err
}
file := fileVName(anchor)
return r.Source, file, nil
func targetToFile(file *spb.VName, p *ppb.DecorationPiece) (*spb.VName, *spb.VName, error) {
return p.GetReference().Source, file, nil
}

// combineDecorPieces combines *ppb.DecorationPieces into a single *srvpb.FileDecorations.
Expand Down Expand Up @@ -398,6 +479,8 @@ func (c *combineDecorPieces) AddInput(accum *srvpb.FileDecorations, p *ppb.Decor
Ticket: kytheuri.ToString(def.Node),
DefinitionLocation: &srvpb.ExpandedAnchor{Ticket: def.Definition.Ticket},
})
case *ppb.DecorationPiece_Diagnostic:
accum.Diagnostic = append(accum.Diagnostic, p.Diagnostic)
default:
panic(fmt.Errorf("unhandled DecorationPiece: %T", p))
}
Expand Down Expand Up @@ -460,25 +543,45 @@ func (c *combineDecorPieces) ExtractOutput(fd *srvpb.FileDecorations) *srvpb.Fil
return fd.Decoration[i].Target < fd.Decoration[j].Target
})
sort.Slice(fd.Target, func(i, j int) bool { return fd.Target[i].Ticket < fd.Target[j].Ticket })

sort.Slice(fd.Diagnostic, func(i, j int) bool {
a, b := fd.Diagnostic[i], fd.Diagnostic[j]
return compare.Compare(a.Span.GetStart().GetByteOffset(), b.Span.GetStart().GetByteOffset()).
AndThen(a.Span.GetEnd().GetByteOffset(), b.Span.GetEnd().GetByteOffset()).
AndThen(a.Message, b.Message) == compare.LT
})
return fd
}

func fileToDecorPiece(src *spb.VName, f *srvpb.File) (*spb.VName, *ppb.DecorationPiece) {
return src, &ppb.DecorationPiece{Piece: &ppb.DecorationPiece_File{f}}
}

func refToDecorPiece(r *ppb.Reference) (*spb.VName, *ppb.DecorationPiece, error) {
_, file, err := toEnclosingFile(r)
if err != nil {
return nil, nil, err
func refToDecorPiece(r *ppb.Reference, emit func(*spb.VName, *ppb.DecorationPiece)) error {
if r.GetKytheKind() == scpb.EdgeKind_TAGGED {
return nil
}
return file, &ppb.DecorationPiece{
p := &ppb.DecorationPiece{
Piece: &ppb.DecorationPiece_Reference{&ppb.Reference{
Source: r.Source,
Kind: r.Kind,
Anchor: r.Anchor,
}},
}, nil
}
file, err := anchorToFileVName(r.Anchor.Ticket)
if err != nil {
return err
}
emit(file, p)
return nil
}

func anchorToFileVName(anchorTicket string) (*spb.VName, error) {
anchor, err := kytheuri.ToVName(anchorTicket)
if err != nil {
return nil, err
}
return fileVName(anchor), nil
}

func fileVName(anchor *spb.VName) *spb.VName {
Expand Down
31 changes: 30 additions & 1 deletion kythe/go/serving/pipeline/beam_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ func TestServingSimpleDecorations(t *testing.T) {
Name: &scpb.Fact_KytheName{scpb.FactName_TEXT_ENCODING},
Value: []byte("ascii"),
}},
Edge: []*scpb.Edge{{
Kind: &scpb.Edge_KytheKind{scpb.EdgeKind_TAGGED},
Target: &spb.VName{Signature: "diagnostic"},
}},
}, {
Source: &spb.VName{Path: "path", Signature: "anchor1"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
Expand Down Expand Up @@ -118,6 +122,19 @@ func TestServingSimpleDecorations(t *testing.T) {
Name: &scpb.Fact_KytheName{scpb.FactName_TEXT},
Value: []byte("def\n"),
}},
}, {
Source: &spb.VName{Signature: "diagnostic"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_DIAGNOSTIC},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_MESSAGE},
Value: []byte("msg"),
}, {
Name: &scpb.Fact_KytheName{scpb.FactName_DETAILS},
Value: []byte("dtails"),
}, {
Name: &scpb.Fact_KytheName{scpb.FactName_CONTEXT_URL},
Value: []byte("https://kythe.io/schema"),
}},
}}

p, s, nodes := ptest.CreateList(testNodes)
Expand Down Expand Up @@ -305,9 +322,21 @@ func TestServingSimpleDecorations(t *testing.T) {
},
}))

// TODO(schroederc): test diagnostics (w/ span)
t.Run("diagnostics", makeDecorTestCase(ctx, xs, &xpb.DecorationsRequest{
Location: &xpb.Location{Ticket: fileTicket},
Diagnostics: true,
}, &xpb.DecorationsReply{
Location: &xpb.Location{Ticket: fileTicket},
Diagnostic: []*cpb.Diagnostic{{
Message: "msg",
Details: "dtails",
ContextUrl: "https://kythe.io/schema",
}},
}))

// TODO(schroederc): test split file contents
// TODO(schroederc): test overrides
// TODO(schroederc): test diagnostics (w/ or w/o span)
}

func TestServingSimpleCrossReferences(t *testing.T) {
Expand Down
76 changes: 76 additions & 0 deletions kythe/go/serving/pipeline/beam_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,82 @@ func TestDecorations_decoration(t *testing.T) {
}
}

func TestDecorations_diagnostics(t *testing.T) {
testNodes := []*scpb.Node{{
Source: &spb.VName{Path: "path", Signature: "anchor1"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_LOC_START},
Value: []byte("5"),
}, {
Name: &scpb.Fact_KytheName{scpb.FactName_LOC_END},
Value: []byte("9"),
}},
Edge: []*scpb.Edge{{
Kind: &scpb.Edge_KytheKind{scpb.EdgeKind_TAGGED},
Target: &spb.VName{Signature: "diagnostic"},
}},
}, {
Source: &spb.VName{Path: "path"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_TEXT},
Value: []byte("some text\n"),
}},
Edge: []*scpb.Edge{{
Kind: &scpb.Edge_KytheKind{scpb.EdgeKind_TAGGED},
Target: &spb.VName{Signature: "diagnostic"},
}},
}, {
Source: &spb.VName{Signature: "diagnostic"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_DIAGNOSTIC},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_MESSAGE},
Value: []byte("msg"),
}, {
Name: &scpb.Fact_KytheName{scpb.FactName_DETAILS},
Value: []byte("deets"),
}, {
Name: &scpb.Fact_KytheName{scpb.FactName_CONTEXT_URL},
Value: []byte("https://kythe.io"),
}},
}}

expected := []*srvpb.FileDecorations{{
File: &srvpb.File{Text: []byte("some text\n")},
Diagnostic: []*cpb.Diagnostic{{
Message: "msg",
Details: "deets",
ContextUrl: "https://kythe.io",
}, {
Span: &cpb.Span{
Start: &cpb.Point{
ByteOffset: 5,
LineNumber: 1,
ColumnOffset: 5,
},
End: &cpb.Point{
ByteOffset: 9,
LineNumber: 1,
ColumnOffset: 9,
},
},
Message: "msg",
Details: "deets",
ContextUrl: "https://kythe.io",
}},
}}

p, s, nodes := ptest.CreateList(testNodes)
decor := FromNodes(s, nodes).Decorations()
debug.Print(s, decor)
passert.Equals(s, beam.DropKey(s, decor), beam.CreateList(s, expected))

if err := ptest.Run(p); err != nil {
t.Fatalf("Pipeline error: %+v", err)
}
}

func TestDecorations_targetDefinition(t *testing.T) {
testNodes := []*scpb.Node{{
Source: &spb.VName{Path: "path", Signature: "anchor1"},
Expand Down
18 changes: 17 additions & 1 deletion kythe/go/serving/pipeline/encoding.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,10 @@ func encodeDecorPiece(file *spb.VName, p *ppb.DecorationPiece, emit func([]byte,
return encodeDecorNode(file, p.Node, emit)
case *ppb.DecorationPiece_Definition_:
return encodeDecorDef(file, p.Definition, emit)
case *ppb.DecorationPiece_Diagnostic:
return encodeDecorDiagnostic(file, p.Diagnostic, emit)
default:
// TODO(schroederc): add diagnostics/overrides
// TODO(schroederc): add overrides
return fmt.Errorf("unknown DecorationPiece: %T", p)
}
}
Expand Down Expand Up @@ -197,6 +199,20 @@ func encodeDecorDef(file *spb.VName, def *ppb.DecorationPiece_Definition, emit f
return nil
}

func encodeDecorDiagnostic(file *spb.VName, d *cpb.Diagnostic, emit func([]byte, []byte)) error {
e, err := columnar.EncodeDecorationsEntry(columnar.DecorationsKeyPrefix, &xspb.FileDecorations{
File: file,
Entry: &xspb.FileDecorations_Diagnostic_{&xspb.FileDecorations_Diagnostic{
Diagnostic: d,
}},
})
if err != nil {
return err
}
emit(e.Key, e.Value)
return nil
}

func encodeEdgesEntry(e *gspb.Edges, emit func([]byte, []byte)) error {
kv, err := gcolumnar.EncodeEdgesEntry(gcolumnar.EdgesKeyPrefix, e)
if err != nil {
Expand Down
Loading

0 comments on commit 0cd5dfc

Please sign in to comment.