Support go modules + VB.Net lexer.

Fixes #201.

_tools/pygments2chroma.py

@@ -16,10 +16,11 @@
 
 import (
     . "github.com/alecthomas/chroma" // nolint
+    "github.com/alecthomas/chroma/lexers/internal"
 )
 
 // {{upper_name}} lexer.
-var {{upper_name}} = Register(MustNewLexer(
+var {{upper_name}} = internal.Register(MustNewLexer(
     &Config{
         Name:      "{{name}}",
         Aliases:   []string{ {{#aliases}}"{{.}}", {{/aliases}} },
@@ -193,4 +194,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

cmd/chroma/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"fmt"
+	"github.com/alecthomas/kong"
 	"io"
 	"io/ioutil"
 	"os"
@@ -13,15 +14,13 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/mattn/go-colorable"
-	"github.com/mattn/go-isatty"
-	"gopkg.in/alecthomas/kingpin.v3-unstable"
-
 	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/formatters"
 	"github.com/alecthomas/chroma/formatters/html"
 	"github.com/alecthomas/chroma/lexers"
 	"github.com/alecthomas/chroma/styles"
+	"github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
 )
 
 var (
@@ -30,34 +29,42 @@ var (
 	commit  = "?"
 	date    = "?"
 
-	profileFlag    = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String()
-	listFlag       = kingpin.Flag("list", "List lexers, styles and formatters.").Bool()
-	unbufferedFlag = kingpin.Flag("unbuffered", "Do not buffer output.").Bool()
-	traceFlag      = kingpin.Flag("trace", "Trace lexer states as they are traversed.").Bool()
-	checkFlag      = kingpin.Flag("check", "Do not format, check for tokenization errors instead.").Bool()
-	filenameFlag   = kingpin.Flag("filename", "Filename to use for selecting a lexer when reading from stdin.").String()
+	description = `
+Chroma is a general purpose syntax highlighting library and corresponding
+command, for Go.
+`
+
+	cli struct {
+		Version    kong.VersionFlag `help:"Show version."`
+		Profile    string           `hidden:"" help:"Enable profiling to file."`
+		List       bool             `help:"List lexers, styles and formatters."`
+		Unbuffered bool             `help:"Do not buffer output."`
+		Trace      bool             `help:"Trace lexer states as they are traversed."`
+		Check      bool             `help:"Do not format, check for tokenization errors instead."`
+		Filename   string           `help:"Filename to use for selecting a lexer when reading from stdin."`
 
-	lexerFlag     = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...)
-	styleFlag     = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
-	formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...)
+		Lexer     string `help:"Lexer to use when formatting." placeholder:"autodetect" short:"l"`
+		Style     string `help:"Style to use for formatting." default:"swapoff" short:"s"`
+		Formatter string `help:"Formatter to use." default:"terminal" short:"f"`
 
-	jsonFlag = kingpin.Flag("json", "Output JSON representation of tokens.").Bool()
+		JSON bool `help:"Output JSON representation of tokens."`
 
-	htmlFlag                      = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool()
-	htmlPrefixFlag                = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String()
-	htmlStylesFlag                = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool()
-	htmlOnlyFlag                  = kingpin.Flag("html-only", "Output HTML fragment.").Bool()
-	htmlInlineStyleFlag           = kingpin.Flag("html-inline-styles", "Output HTML with inline styles (no classes).").Bool()
-	htmlTabWidthFlag              = kingpin.Flag("html-tab-width", "Set the HTML tab width.").Default("8").Int()
-	htmlLinesFlag                 = kingpin.Flag("html-lines", "Include line numbers in output.").Bool()
-	htmlLinesTableFlag            = kingpin.Flag("html-lines-table", "Split line numbers and code in a HTML table").Bool()
-	htmlLinesStyleFlag            = kingpin.Flag("html-lines-style", "Style for line numbers.").String()
-	htmlHighlightFlag             = kingpin.Flag("html-highlight", "Highlight these lines.").PlaceHolder("N[:M][,...]").String()
-	htmlHighlightStyleFlag        = kingpin.Flag("html-highlight-style", "Style used for highlighting lines.").String()
-	htmlBaseLineFlag              = kingpin.Flag("html-base-line", "Base line number.").Default("1").Int()
-	htmlPreventSurroundingPreFlag = kingpin.Flag("html-prevent-surrounding-pre", "Prevent the surrounding pre tag.").Bool()
+		HTML                      bool   `help:"Enable HTML mode (equivalent to '--formatter html')."`
+		HTMLPrefix                string `help:"HTML CSS class prefix." placeholder:"PREFIX"`
+		HTMLStyles                bool   `help:"Output HTML CSS styles."`
+		HTMLOnly                  bool   `help:"Output HTML fragment."`
+		HTMLInlineStyles          bool   `help:"Output HTML with inline styles (no classes)."`
+		HTMLTabWidth              int    `help:"Set the HTML tab width." default:"8"`
+		HTMLLines                 bool   `help:"Include line numbers in output."`
+		HTMLLinesTable            bool   `help:"Split line numbers and code in a HTML table"`
+		HTMLLinesStyle            string `help:"Style for line numbers."`
+		HTMLHighlight             string `help:"Highlight these lines." placeholder:"N[:M][,...]"`
+		HTMLHighlightStyle        string `help:"Style used for highlighting lines."`
+		HTMLBaseLine              int    `help:"Base line number." default:"1"`
+		HTMLPreventSurroundingPre bool   `help:"Prevent the surrounding pre tag."`
 
-	filesArgs = kingpin.Arg("files", "Files to highlight.").ExistingFiles()
+		Files []string `arg:"" help:"Files to highlight." type:"existingfile"`
+	}
 )
 
 type flushableWriter interface {
@@ -70,19 +77,16 @@ type nopFlushableWriter struct{ io.Writer }
 func (n *nopFlushableWriter) Flush() error { return nil }
 
 func main() {
-	kingpin.CommandLine.Version(fmt.Sprintf("%s-%s-%s", version, commit, date))
-	kingpin.CommandLine.Help = `
-Chroma is a general purpose syntax highlighting library and corresponding
-command, for Go.
-`
-	kingpin.Parse()
-	if *listFlag {
+	ctx := kong.Parse(&cli, kong.Description(description), kong.Vars{
+		"version": fmt.Sprintf("%s-%s-%s", version, commit, date),
+	}, )
+	if cli.List {
 		listAll()
 		return
 	}
-	if *profileFlag != "" {
-		f, err := os.Create(*profileFlag)
-		kingpin.FatalIfError(err, "")
+	if cli.Profile != "" {
+		f, err := os.Create(cli.Profile)
+		ctx.FatalIfErrorf(err)
 		pprof.StartCPUProfile(f)
 		signals := make(chan os.Signal, 1)
 		signal.Notify(signals, os.Interrupt)
@@ -99,94 +103,94 @@ command, for Go.
 		out = colorable.NewColorableStdout()
 	}
 	var w flushableWriter
-	if *unbufferedFlag {
+	if cli.Unbuffered {
 		w = &nopFlushableWriter{out}
 	} else {
 		w = bufio.NewWriterSize(out, 16384)
 	}
 	defer w.Flush()
 
-	if *jsonFlag {
-		*formatterFlag = "json"
+	if cli.JSON {
+		cli.Formatter = "json"
 	}
 
-	if *htmlFlag {
-		*formatterFlag = "html"
+	if cli.HTML {
+		cli.Formatter = "html"
 	}
 
 	// Retrieve user-specified style, clone it, and add some overrides.
-	builder := styles.Get(*styleFlag).Builder()
-	if *htmlHighlightStyleFlag != "" {
-		builder.Add(chroma.LineHighlight, *htmlHighlightStyleFlag)
+	builder := styles.Get(cli.Style).Builder()
+	if cli.HTMLHighlightStyle != "" {
+		builder.Add(chroma.LineHighlight, cli.HTMLHighlightStyle)
 	}
-	if *htmlLinesStyleFlag != "" {
-		builder.Add(chroma.LineNumbers, *htmlLinesStyleFlag)
+	if cli.HTMLLinesStyle != "" {
+		builder.Add(chroma.LineNumbers, cli.HTMLLinesStyle)
 	}
 	style, err := builder.Build()
-	kingpin.FatalIfError(err, "")
+	ctx.FatalIfErrorf(err)
 
 	// Dump styles.
-	if *htmlStylesFlag {
+	if cli.HTMLStyles {
 		formatter := html.New(html.WithClasses())
 		formatter.WriteCSS(w, style)
 		return
 	}
 
-	if *formatterFlag == "html" {
+	if cli.Formatter == "html" {
 		options := []html.Option{
-			html.TabWidth(*htmlTabWidthFlag),
-			html.BaseLineNumber(*htmlBaseLineFlag),
+			html.TabWidth(cli.HTMLTabWidth),
+			html.BaseLineNumber(cli.HTMLBaseLine),
 		}
-		if *htmlPrefixFlag != "" {
-			options = append(options, html.ClassPrefix(*htmlPrefixFlag))
+		if cli.HTMLPrefix != "" {
+			options = append(options, html.ClassPrefix(cli.HTMLPrefix))
 		}
-		if !*htmlInlineStyleFlag {
+		if !cli.HTMLInlineStyles {
 			options = append(options, html.WithClasses())
 		}
-		if !*htmlOnlyFlag {
+		if !cli.HTMLOnly {
 			options = append(options, html.Standalone())
 		}
-		if *htmlLinesFlag {
+		if cli.HTMLLines {
 			options = append(options, html.WithLineNumbers())
 		}
-		if *htmlLinesTableFlag {
+		if cli.HTMLLinesTable {
 			options = append(options, html.LineNumbersInTable())
 		}
-		if *htmlPreventSurroundingPreFlag {
+		if cli.HTMLPreventSurroundingPre {
 			options = append(options, html.PreventSurroundingPre())
 		}
-		if len(*htmlHighlightFlag) > 0 {
+		if len(cli.HTMLHighlight) > 0 {
 			ranges := [][2]int{}
-			for _, span := range strings.Split(*htmlHighlightFlag, ",") {
+			for _, span := range strings.Split(cli.HTMLHighlight, ",") {
 				parts := strings.Split(span, ":")
 				if len(parts) > 2 {
-					kingpin.Fatalf("range should be N[:M], not %q", span)
+					ctx.Fatalf("range should be N[:M], not %q", span)
 				}
 				start, err := strconv.ParseInt(parts[0], 10, 64)
-				kingpin.FatalIfError(err, "min value of range should be integer not %q", parts[0])
+				ctx.FatalIfErrorf(err, "min value of range should be integer not %q", parts[0])
 				end := start
 				if len(parts) == 2 {
 					end, err = strconv.ParseInt(parts[1], 10, 64)
-					kingpin.FatalIfError(err, "max value of range should be integer not %q", parts[1])
+					ctx.FatalIfErrorf(err, "max value of range should be integer not %q", parts[1])
 				}
 				ranges = append(ranges, [2]int{int(start), int(end)})
 			}
 			options = append(options, html.HighlightLines(ranges))
 		}
 		formatters.Register("html", html.New(options...))
 	}
-	if len(*filesArgs) == 0 {
+	if len(cli.Files) == 0 {
 		contents, err := ioutil.ReadAll(os.Stdin)
-		kingpin.FatalIfError(err, "")
-		format(w, style, lex(*filenameFlag, string(contents)))
+		ctx.FatalIfErrorf(err)
+		format(ctx, w, style, lex(ctx, cli.Filename, string(contents)))
 	} else {
-		for _, filename := range *filesArgs {
+		for _, filename := range cli.Files {
 			contents, err := ioutil.ReadFile(filename)
-			kingpin.FatalIfError(err, "")
-			if *checkFlag {
-				check(filename, lex(filename, string(contents)))
+			ctx.FatalIfErrorf(err)
+			if cli.Check {
+				check(filename, lex(ctx, filename, string(contents)))
 			} else {
-				format(w, style, lex(filename, string(contents)))
+				format(ctx, w, style, lex(ctx, filename, string(contents)))
 			}
 		}
 	}
@@ -224,23 +228,23 @@ func listAll() {
 	fmt.Println()
 }
 
-func lex(path string, contents string) chroma.Iterator {
+func lex(ctx *kong.Context, path string, contents string) chroma.Iterator {
 	lexer := selexer(path, contents)
 	if lexer == nil {
 		lexer = lexers.Fallback
 	}
 	if rel, ok := lexer.(*chroma.RegexLexer); ok {
-		rel.Trace(*traceFlag)
+		rel.Trace(cli.Trace)
 	}
 	lexer = chroma.Coalesce(lexer)
 	it, err := lexer.Tokenise(nil, string(contents))
-	kingpin.FatalIfError(err, "")
+	ctx.FatalIfErrorf(err)
 	return it
 }
 
 func selexer(path, contents string) (lexer chroma.Lexer) {
-	if *lexerFlag != "" {
-		return lexers.Get(*lexerFlag)
+	if cli.Lexer != "" {
+		return lexers.Get(cli.Lexer)
 	}
 	if path != "" {
 		lexer := lexers.Match(path)
@@ -251,10 +255,10 @@ func selexer(path, contents string) (lexer chroma.Lexer) {
 	return lexers.Analyse(contents)
 }
 
-func format(w io.Writer, style *chroma.Style, it chroma.Iterator) {
-	formatter := formatters.Get(*formatterFlag)
+func format(ctx *kong.Context, w io.Writer, style *chroma.Style, it chroma.Iterator) {
+	formatter := formatters.Get(cli.Formatter)
 	err := formatter.Format(w, style, it)
-	kingpin.FatalIfError(err, "")
+	ctx.FatalIfErrorf(err)
 }
 
 func check(filename string, it chroma.Iterator) {

delegate_test.go

@@ -1,7 +1,6 @@
 package chroma
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/alecthomas/assert"
@@ -105,7 +104,6 @@ func TestDelegate(t *testing.T) {
 			it, err := delegate.Tokenise(nil, test.source)
 			assert.NoError(t, err)
 			actual := it.Tokens()
-			fmt.Println(actual)
 			assert.Equal(t, test.expected, actual)
 		})
 	}

go.mod

@@ -0,0 +1,14 @@
+module github.com/alecthomas/chroma
+
+require (
+	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
+	github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 // indirect
+	github.com/alecthomas/kong v0.1.15
+	github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 // indirect
+	github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964
+	github.com/dlclark/regexp2 v1.1.6
+	github.com/mattn/go-colorable v0.0.9
+	github.com/mattn/go-isatty v0.0.4
+	github.com/sergi/go-diff v1.0.0 // indirect
+	golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
+)

go.sum

@@ -0,0 +1,26 @@
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/kong v0.1.15 h1:IWBg+KrLvoHBicD50OzMI8fKjrtAa1okMR9g38HVM/s=
+github.com/alecthomas/kong v0.1.15/go.mod h1:0m2VYms8rH0qbCqVB2gvGHk74bqLIq0HXjCs5bNbNQU=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY=
+github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
+github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
+github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 h1:YAFjXN64LMvktoUZH9zgY4lGc/msGN7HQfoSuKCgaDU=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

lexers/testdata/vbnet.actual

@@ -0,0 +1,7 @@
+Imports System
+
+Public Module Hello
+   Public Sub Main(  )
+      Console.WriteLine("hello, world")
+   End Sub
+End Module