Compiler fixes (#4215)

* use GlobalString to get linguist-path

* don't convert if there's no scope name

* write version file out in JSON output

* new source for version info

It's actually 6.4.0, but I'm making a follow-up commit to bump "back" up to
6.3.9 so we have a single commit bumping the new version file usable as example
in CONTRIBUTING.md.  It makes sense, I swear.

* bump to 6.4.0

* use new commit as example in CONTRIBUTING.md

* fetch version from new VERSION file

CONTRIBUTING.md

@@ -151,7 +151,7 @@ If you are the current maintainer of this gem:
   1. Bump the Gemfile and Gemfile.lock versions for an app which relies on this gem
   1. Install the new gem locally
   1. Test behavior locally, branch deploy, whatever needs to happen
-1. Bump gem version in `lib/linguist/version.rb`, [like this](https://github.com/github/linguist/commit/8d2ea90a5ba3b2fe6e1508b7155aa4632eea2985).
+1. Bump gem version in `lib/linguist/VERSION`, [like this](https://github.com/github/linguist/commit/3212355400974ce5f7873a71eb8b85b1c5f4a6d2).
 1. Make a PR to github/linguist, [like this](https://github.com/github/linguist/pull/1238).
 1. Build a local gem: `bundle exec rake build_gem`
 1. Merge github/linguist PR

lib/linguist/VERSION

@@ -0,0 +1 @@
+6.4.0

lib/linguist/version.rb

@@ -1,3 +1,3 @@
 module Linguist
-  VERSION = "6.4.0"
+  VERSION = File.read(File.expand_path("../VERSION", __FILE__)).strip
 end

tools/grammars/cmd/grammar-compiler/main.go

@@ -40,7 +40,7 @@ func main() {
 				},
 			},
 			Action: func(c *cli.Context) error {
-				conv, err := compiler.NewConverter(c.String("linguist-path"))
+				conv, err := compiler.NewConverter(c.GlobalString("linguist-path"))
 				if err != nil {
 					return wrap(err)
 				}
@@ -65,7 +65,7 @@ func main() {
 				},
 			},
 			Action: func(c *cli.Context) error {
-				conv, err := compiler.NewConverter(c.String("linguist-path"))
+				conv, err := compiler.NewConverter(c.GlobalString("linguist-path"))
 				if err != nil {
 					return wrap(err)
 				}
@@ -91,7 +91,7 @@ func main() {
 				cli.StringFlag{Name: "out, o"},
 			},
 			Action: func(c *cli.Context) error {
-				conv, err := compiler.NewConverter(c.String("linguist-path"))
+				conv, err := compiler.NewConverter(c.GlobalString("linguist-path"))
 				if err != nil {
 					return cli.NewExitError(err, 1)
 				}

tools/grammars/compiler/converter.go

@@ -18,7 +18,8 @@ import (
 )
 
 type Converter struct {
-	root string
+	root    string
+	version string
 
 	modified bool
 	grammars map[string][]string
@@ -194,6 +195,13 @@ func (conv *Converter) WriteJSON(rulePath string) error {
 		return err
 	}
 
+	f, err := os.Create(path.Join(rulePath, "version"))
+	if err != nil {
+		return err
+	}
+	f.Write([]byte(conv.version))
+	f.Close()
+
 	for _, repo := range conv.Loaded {
 		for scope, file := range repo.Files {
 			p := path.Join(rulePath, scope+".json")
@@ -249,12 +257,17 @@ func (conv *Converter) Report() error {
 }
 
 func NewConverter(root string) (*Converter, error) {
+	ver, err := ioutil.ReadFile(path.Join(root, "lib", "linguist", "VERSION"))
+	if err != nil {
+		return nil, err
+	}
+
 	yml, err := ioutil.ReadFile(path.Join(root, "grammars.yml"))
 	if err != nil {
 		return nil, err
 	}
 
-	conv := &Converter{root: root}
+	conv := &Converter{root: root, version: strings.TrimSpace(string(ver))}
 
 	if err := yaml.Unmarshal(yml, &conv.grammars); err != nil {
 		return nil, err

tools/grammars/compiler/errors.go

@@ -83,3 +83,13 @@ func (err *InvalidRegexError) Error() string {
 		"Invalid regex in grammar: %s contains a malformed regex (%s)",
 		err.File, err.Err)
 }
+
+type UndeclaredScopeError struct {
+	Path string
+}
+
+func (err *UndeclaredScopeError) Error() string {
+	return fmt.Sprintf(
+		"Undeclared scope in grammar: `%s` has no scope name",
+		err.Path)
+}

tools/grammars/compiler/loader_fs.go

@@ -73,7 +73,7 @@ func (l *fsLoader) load() {
 			path = rel
 		}
 
-		rule, unknown, err := ConvertProto(filepath.Ext(path), data)
+		rule, unknown, err := ConvertProto(path, filepath.Ext(path), data)
 		if err != nil {
 			l.Fail(&ConversionError{path, err})
 			continue

tools/grammars/compiler/loader_url.go

@@ -41,7 +41,7 @@ func (l *urlLoader) loadTarball(r io.Reader) {
 			}
 
 			ext := filepath.Ext(header.Name)
-			rule, unknown, err := ConvertProto(ext, data)
+			rule, unknown, err := ConvertProto(header.Name, ext, data)
 			if err != nil {
 				l.Fail(&ConversionError{header.Name, err})
 				continue
@@ -77,7 +77,7 @@ func (l *urlLoader) load() {
 
 	ext := filepath.Ext(l.Source)
 	filename := filepath.Base(l.Source)
-	rule, unknown, err := ConvertProto(ext, data)
+	rule, unknown, err := ConvertProto(l.Source, ext, data)
 	if err != nil {
 		l.Fail(&ConversionError{filename, err})
 		return

tools/grammars/compiler/proto.go

@@ -49,7 +49,7 @@ func filterUnusedKeys(keys []string) (out []string) {
 	return
 }
 
-func ConvertProto(ext string, data []byte) (*grammar.Rule, []string, error) {
+func ConvertProto(path, ext string, data []byte) (*grammar.Rule, []string, error) {
 	var (
 		raw map[string]interface{}
 		out grammar.Rule
@@ -92,5 +92,9 @@ func ConvertProto(ext string, data []byte) (*grammar.Rule, []string, error) {
 		return nil, nil, err
 	}
 
+	if out.ScopeName == "" {
+		return nil, nil, &UndeclaredScopeError{path}
+	}
+
 	return &out, filterUnusedKeys(md.Unused), nil
 }