Skip to content

Commit

Permalink
Removed NilAnnotator and amended data structures to match new API.
Browse files Browse the repository at this point in the history
  • Loading branch information
gbbr committed May 12, 2015
1 parent 47c78b3 commit 875edca
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 208 deletions.
111 changes: 5 additions & 106 deletions highlight.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,13 @@ package syntaxhighlight

import (
"bytes"
"html"
"io"
"strings"
"text/scanner"
"text/template"
"unicode"
"unicode/utf8"

"github.com/sourcegraph/annotate"
"sourcegraph.com/sourcegraph/go-sourcegraph/sourcegraph"
"sourcegraph.com/sourcegraph/vcsstore/vcsclient"
)

// Kind represents a syntax highlighting kind (class) which will be assigned to tokens.
Expand Down Expand Up @@ -57,11 +53,12 @@ type HTMLConfig struct {
HTMLAttrName string
HTMLAttrValue string
Decimal string
Whitespace string
}

type HTMLPrinter HTMLConfig

func (c HTMLConfig) class(kind Kind) string {
func (c HTMLConfig) Class(kind Kind) string {
switch kind {
case String:
return c.String
Expand Down Expand Up @@ -92,7 +89,7 @@ func (c HTMLConfig) class(kind Kind) string {
}

func (p HTMLPrinter) Print(w io.Writer, kind Kind, tokText string) error {
class := ((HTMLConfig)(p)).class(kind)
class := ((HTMLConfig)(p)).Class(kind)
if class != "" {
_, err := w.Write([]byte(`<span class="`))
if err != nil {
Expand Down Expand Up @@ -121,109 +118,10 @@ type Annotator interface {
Annotate(start int, kind Kind, tokText string) (*annotate.Annotation, error)
}

// NilAnnotator is a special kind of annotator that always returns nil, but stores
// within itself the snippet of source code that is passed through it as tokens.
//
// This functionality is useful when one wishes to obtain the tokenized source as a data
// structure, as opposed to an annotated string, allowing full control over rendering and
// displaying it.
type NilAnnotator struct {
Config HTMLConfig
Code *sourcegraph.SourceCode
byteOffset int
}

func NewNilAnnotator(e *vcsclient.FileWithRange) *NilAnnotator {
ann := NilAnnotator{
Config: DefaultHTMLConfig,
Code: &sourcegraph.SourceCode{
Lines: make([]*sourcegraph.SourceCodeLine, 0, bytes.Count(e.Contents, []byte("\n"))),
},
byteOffset: int(e.StartByte),
}
ann.addLine(ann.byteOffset)
return &ann
}

func (a *NilAnnotator) addToken(t interface{}) {
line := a.Code.Lines[len(a.Code.Lines)-1]
if line.Tokens == nil {
line.Tokens = make([]interface{}, 0, 1)
}
// If this token and the previous one are both strings, merge them.
n := len(line.Tokens)
if t1, ok := t.(string); ok && n > 0 {
if t2, ok := (line.Tokens[n-1]).(string); ok {
line.Tokens[n-1] = string(t1 + t2)
return
}
}
line.Tokens = append(line.Tokens, t)
}

func (a *NilAnnotator) addLine(startByte int) {
a.Code.Lines = append(a.Code.Lines, &sourcegraph.SourceCodeLine{StartByte: startByte})
if len(a.Code.Lines) > 1 {
lastLine := a.Code.Lines[len(a.Code.Lines)-2]
lastLine.EndByte = startByte - 1
}
}

func (a *NilAnnotator) addMultilineToken(startByte int, unsafeHTML string, class string) {
lines := strings.Split(unsafeHTML, "\n")
for n, unsafeHTML := range lines {
if len(unsafeHTML) > 0 {
a.addToken(&sourcegraph.SourceCodeToken{
StartByte: startByte,
EndByte: startByte + len(unsafeHTML),
Class: class,
Label: html.EscapeString(unsafeHTML),
})
startByte += len(unsafeHTML)
}
if n < len(lines)-1 {
a.addLine(startByte)
}
}
}

func (a *NilAnnotator) Annotate(start int, kind Kind, tokText string) (*annotate.Annotation, error) {
class := ((HTMLConfig)(a.Config)).class(kind)
txt := html.EscapeString(tokText)
start += a.byteOffset

switch {
// New line
case tokText == "\n":
a.addLine(start + 1)

// Whitespace token
case class == "":
a.addToken(txt)

// Multiline token (ie. block comments, string literals)
case strings.Contains(tokText, "\n"):
// Here we pass the unescaped string so we can calculate line lenghts correctly.
// This method is expected to take responsibility of escaping any token text.
a.addMultilineToken(start+1, tokText, class)

// Token
default:
a.addToken(&sourcegraph.SourceCodeToken{
StartByte: start,
EndByte: start + len(tokText),
Class: class,
Label: txt,
})
}

return nil, nil
}

type HTMLAnnotator HTMLConfig

func (a HTMLAnnotator) Annotate(start int, kind Kind, tokText string) (*annotate.Annotation, error) {
class := ((HTMLConfig)(a)).class(kind)
class := ((HTMLConfig)(a)).Class(kind)
if class != "" {
left := []byte(`<span class="`)
left = append(left, []byte(class)...)
Expand Down Expand Up @@ -251,6 +149,7 @@ var DefaultHTMLConfig = HTMLConfig{
HTMLAttrName: "atn",
HTMLAttrValue: "atv",
Decimal: "dec",
Whitespace: "",
}

func Print(s *scanner.Scanner, w io.Writer, p Printer) error {
Expand Down
102 changes: 0 additions & 102 deletions highlight_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import (

"github.com/kr/pretty"
"github.com/sourcegraph/annotate"
"sourcegraph.com/sourcegraph/go-sourcegraph/sourcegraph"
"sourcegraph.com/sourcegraph/vcsstore/vcsclient"
)

var saveExp = flag.Bool("exp", false, "overwrite all expected output files with actual output (returning a failure)")
Expand Down Expand Up @@ -99,106 +97,6 @@ func TestAnnotate(t *testing.T) {
}
}

// codeEquals tests the equality between the given SourceCode entry and an
// array of lines containing arrays of tokens as their string representation.
func codeEquals(code *sourcegraph.SourceCode, want [][]string) bool {
if len(code.Lines) != len(want) {
return false
}
for i, line := range code.Lines {
for j, t := range line.Tokens {
switch t := t.(type) {
case *sourcegraph.SourceCodeToken:
if t.Label != want[i][j] {
return false
}
case string:
if t != want[i][j] {
return false
}
}
}
}
return true
}

func TestCodeEquals(t *testing.T) {
for _, tt := range []struct {
code *sourcegraph.SourceCode
want [][]string
}{
{
code: &sourcegraph.SourceCode{
Lines: []*sourcegraph.SourceCodeLine{
&sourcegraph.SourceCodeLine{
Tokens: []interface{}{
&sourcegraph.SourceCodeToken{Label: "a"},
&sourcegraph.SourceCodeToken{Label: "b"},
"c",
&sourcegraph.SourceCodeToken{Label: "d"},
"e",
},
},
&sourcegraph.SourceCodeLine{},
&sourcegraph.SourceCodeLine{
Tokens: []interface{}{
"c",
},
},
},
},
want: [][]string{[]string{"a", "b", "c", "d", "e"}, []string{}, []string{"c"}},
},
} {
if !codeEquals(tt.code, tt.want) {
t.Errorf("Expected: %# v, Got: %# v\n", tt.code, tt.want)
}
}
}

func newFileWithRange(src []byte) *vcsclient.FileWithRange {
return &vcsclient.FileWithRange{
TreeEntry: &vcsclient.TreeEntry{Contents: []byte(src)},
FileRange: vcsclient.FileRange{StartByte: 0, EndByte: int64(len(src))},
}
}

func TestNilAnnotator_multiLineTokens(t *testing.T) {
for _, tt := range []struct {
src string
want [][]string
}{
{
src: "/* I am\na multiline\ncomment\n*/",
want: [][]string{
[]string{"/* I am"},
[]string{"a multiline"},
[]string{"comment"},
[]string{"*/"},
},
},
{
src: "a := `I am\na multiline\nstring literal\n`",
want: [][]string{
[]string{"a", " ", ":", "=", " ", "`I am"},
[]string{"a multiline"},
[]string{"string literal"},
[]string{"`"},
},
},
} {
e := newFileWithRange([]byte(tt.src))
ann := NewNilAnnotator(e)
_, err := Annotate(e.Contents, ann)
if err != nil {
t.Fatal(err)
}
if !codeEquals(ann.Code, tt.want) {
t.Errorf("Expected %# v\n\nGot %# v", tt.want, pretty.Formatter(ann.Code.Lines))
}
}
}

func BenchmarkAnnotate(b *testing.B) {
input, err := ioutil.ReadFile("testdata/net_http_client.go")
if err != nil {
Expand Down

0 comments on commit 875edca

Please sign in to comment.