diff --git a/Gopkg.lock b/Gopkg.lock index ecdbcf2b9..53aac510c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -153,6 +153,12 @@ revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" version = "v1.2.1" +[[projects]] + branch = "master" + name = "github.com/toqueteos/trie" + packages = ["."] + revision = "56fed4a05683322f125e2d78ee269bb102280392" + [[projects]] branch = "master" name = "github.com/xanzy/ssh-agent" @@ -235,6 +241,16 @@ packages = ["."] revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec" +[[projects]] + name = "gopkg.in/src-d/enry.v1" + packages = [ + ".", + "data", + "internal/tokenizer" + ] + revision = "0db3b4b5536e6dc4d9109d42897c00a5d92af0a7" + version = "v1.6.3" + [[projects]] name = "gopkg.in/src-d/go-billy.v4" packages = [ @@ -346,6 +362,12 @@ ] revision = "2cb632cdef3c332f5cba7f035479213ca31dfe71" +[[projects]] + name = "gopkg.in/toqueteos/substring.v1" + packages = ["."] + revision = "c5f61671513240ddf5563635cc4a90e9f3ae4710" + version = "v1.0.2" + [[projects]] name = "gopkg.in/warnings.v0" packages = ["."] @@ -355,6 +377,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b8d10024b06f5ff4a62881120cebb22b4640d8ab28edaf5bbd87fd6fd1ff520c" + inputs-digest = "e416a3f7e1bac3a8e47d58166543ab89fe960a76af82f5f5263dd11b34c77e5b" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index f32864bd3..f3f692839 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -41,3 +41,7 @@ [[override]] name = "github.com/satori/go.uuid" revision = "36e9d2ebbde5e3f13ab2e25625fd453271d6522e" + +[[constraint]] + name = "gopkg.in/src-d/enry.v1" + version = "1.6.3" diff --git a/internal/function/language.go b/internal/function/language.go new file mode 100644 index 000000000..8926d27c5 --- /dev/null +++ b/internal/function/language.go @@ -0,0 +1,120 @@ +package function + +import ( + "fmt" + + enry "gopkg.in/src-d/enry.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" +) + +// Language gets the language of a file given its path and +// the optional content of the file. +type Language struct { + Left sql.Expression + Right sql.Expression +} + +// NewLanguage creates a new Language UDF. +func NewLanguage(args ...sql.Expression) (sql.Expression, error) { + var left, right sql.Expression + switch len(args) { + case 1: + left = args[0] + case 2: + left = args[0] + right = args[1] + default: + return nil, sql.ErrInvalidArgumentNumber.New("1 or 2", len(args)) + } + + return &Language{left, right}, nil +} + +// Resolved implements the Expression interface. +func (f *Language) Resolved() bool { + return f.Left.Resolved() && (f.Right == nil || f.Right.Resolved()) +} + +func (f *Language) String() string { + if f.Right == nil { + return fmt.Sprintf("language(%s)", f.Left) + } + return fmt.Sprintf("language(%s, %s)", f.Left, f.Right) +} + +// IsNullable implements the Expression interface. +func (f *Language) IsNullable() bool { + return f.Left.IsNullable() || (f.Right != nil && f.Right.IsNullable()) +} + +// Type implements the Expression interface. +func (Language) Type() sql.Type { + return sql.Text +} + +// TransformUp implements the Expression interface. +func (f *Language) TransformUp(fn sql.TransformExprFunc) (sql.Expression, error) { + left, err := f.Left.TransformUp(fn) + if err != nil { + return nil, err + } + + var right sql.Expression + if f.Right != nil { + right, err = f.Right.TransformUp(fn) + if err != nil { + return nil, err + } + } + + return fn(&Language{left, right}) +} + +// Eval implements the Expression interface. +func (f *Language) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + left, err := f.Left.Eval(ctx, row) + if err != nil { + return nil, err + } + + if left == nil { + return nil, nil + } + + left, err = sql.Text.Convert(left) + if err != nil { + return nil, err + } + + path := left.(string) + var blob []byte + + if f.Right != nil { + right, err := f.Right.Eval(ctx, row) + if err != nil { + return nil, err + } + + if right == nil { + return nil, nil + } + + right, err = sql.Blob.Convert(right) + if err != nil { + return nil, err + } + + blob = right.([]byte) + } + + return enry.GetLanguage(path, blob), nil +} + +// Children implements the Expression interface. +func (f *Language) Children() []sql.Expression { + if f.Right == nil { + return []sql.Expression{f.Left} + } + + return []sql.Expression{f.Left, f.Right} +} diff --git a/internal/function/language_test.go b/internal/function/language_test.go new file mode 100644 index 000000000..dfb6cabc7 --- /dev/null +++ b/internal/function/language_test.go @@ -0,0 +1,53 @@ +package function + +import ( + "testing" + + "github.com/stretchr/testify/require" + errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-mysql-server.v0/sql" + "gopkg.in/src-d/go-mysql-server.v0/sql/expression" +) + +func TestLanguage(t *testing.T) { + testCases := []struct { + name string + row sql.Row + expected interface{} + err *errors.Kind + }{ + {"left is null", sql.NewRow(nil), nil, nil}, + {"right is null", sql.NewRow("foo", nil), nil, nil}, + {"both are null", sql.NewRow(nil, nil), nil, nil}, + {"only path is given", sql.NewRow("foo.rb"), "Ruby", nil}, + {"too many args given", sql.NewRow("foo.rb", "bar", "baz"), nil, sql.ErrInvalidArgumentNumber}, + {"path and blob are given", sql.NewRow("foo", "#!/usr/bin/env python\n\nprint 'foo'"), "Python", nil}, + {"invalid blob type given", sql.NewRow("foo", 5), nil, sql.ErrInvalidType}, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + + var args = make([]sql.Expression, len(tt.row)) + for i := range tt.row { + args[i] = expression.NewGetField(i, sql.Text, "", false) + } + + f, err := NewLanguage(args...) + if err == nil { + var val interface{} + val, err = f.Eval(nil, tt.row) + if tt.err == nil { + require.NoError(err) + require.Equal(tt.expected, val) + } + } + + if tt.err != nil { + require.Error(err) + require.True(tt.err.Is(err)) + } + }) + } +} diff --git a/internal/function/registry.go b/internal/function/registry.go index 84cd99b3f..8ca2da6ba 100644 --- a/internal/function/registry.go +++ b/internal/function/registry.go @@ -8,4 +8,5 @@ var Functions = sql.Functions{ "commit_has_blob": sql.Function2(NewCommitHasBlob), "history_idx": sql.Function2(NewHistoryIdx), "commit_has_tree": sql.Function2(NewCommitHasTree), + "language": sql.FunctionN(NewLanguage), } diff --git a/vendor/github.com/toqueteos/trie/LICENSE.txt b/vendor/github.com/toqueteos/trie/LICENSE.txt new file mode 100644 index 000000000..8a21d8403 --- /dev/null +++ b/vendor/github.com/toqueteos/trie/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2013 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/toqueteos/trie/README.md b/vendor/github.com/toqueteos/trie/README.md new file mode 100644 index 000000000..5ba218b2e --- /dev/null +++ b/vendor/github.com/toqueteos/trie/README.md @@ -0,0 +1,8 @@ +# Trie + +This is a Go package implementing a trie (prefix tree) over byte slices. + +This is a temporary naive implementation that will be replaced with a double-array trie (see +https://github.com/cespare/go-trie). The API will hopefully remain unchanged. + +Package documentation may be found on [godoc.org](http://godoc.org/github.com/cespare/trie). diff --git a/vendor/github.com/toqueteos/trie/example_test.go b/vendor/github.com/toqueteos/trie/example_test.go new file mode 100644 index 000000000..dab81dab9 --- /dev/null +++ b/vendor/github.com/toqueteos/trie/example_test.go @@ -0,0 +1,66 @@ +package trie_test + +import ( + "fmt" + + "github.com/cespare/trie" +) + +// This example shows simple usage of a Trie as a []byte set. +func ExampleTrie() { + t := trie.New() + t.Insert([]byte("hello")) + t.Insert([]byte("world")) + for _, s := range []string{"hello", "world", "he", "h", "worlds", ""} { + fmt.Println(t.Contains([]byte(s))) + } + // Output: + // true + // true + // false + // false + // false + // false +} + +// This example demonstrates walking through the nodes of a Trie. +func ExampleNode() { + t := trie.New() + for _, s := range []string{"folk", "foxes", "fox"} { + t.Insert([]byte(s)) + } + n := t.Root() + fmt.Println(n.Terminal()) // false + next, ok := n.Walk('a') + fmt.Println(ok) // false + for _, c := range []byte("fox") { + next, ok = n.Walk(c) + if !ok { + panic("unexpected") + } + fmt.Println(ok) // true + n = next + } + fmt.Println(n.Terminal()) // true + fmt.Println(n.Leaf()) // false + for _, c := range []byte("es") { + next, ok = n.Walk(c) + if !ok { + panic("unexpected") + } + n = next + } + fmt.Println(n.Terminal()) // true + fmt.Println(n.Leaf()) // true + + // Output: + // false + // false + // true + // true + // true + // true + // false + // true + // true +} diff --git a/vendor/github.com/toqueteos/trie/trie.go b/vendor/github.com/toqueteos/trie/trie.go new file mode 100644 index 000000000..4411f3e42 --- /dev/null +++ b/vendor/github.com/toqueteos/trie/trie.go @@ -0,0 +1,102 @@ +// Package trie is an implementation of a trie (prefix tree) data structure over byte slices. It provides a +// small and simple API for usage as a set as well as a 'Node' API for walking the trie. +package trie + +// A Trie is a a prefix tree. +type Trie struct { + root *Node +} + +// New construct a new, empty Trie ready for use. +func New() *Trie { + return &Trie{ + root: &Node{}, + } +} + +// Insert puts b into the Trie. It returns true if the element was not previously in t. +func (t *Trie) Insert(b []byte) bool { + n := t.root + for _, c := range b { + next, ok := n.Walk(c) + if !ok { + next = &Node{} + n.branches[c] = next + n.hasChildren = true + } + n = next + } + if n.terminal { + return false + } + n.terminal = true + return true +} + +// Contains checks t for membership of b. +func (t *Trie) Contains(b []byte) bool { + n := t.root + for _, c := range b { + next, ok := n.Walk(c) + if !ok { + return false + } + n = next + } + return n.terminal +} + +// PrefixIndex walks through `b` until a prefix is found (terminal node) or it is exhausted. +func (t *Trie) PrefixIndex(b []byte) int { + var idx int + n := t.root + for _, c := range b { + next, ok := n.Walk(c) + if !ok { + return -1 + } + if next.terminal { + return idx + } + n = next + idx++ + } + if !n.terminal { + idx = -1 + } + return idx +} + +// Root returns the root node of a Trie. A valid Trie (i.e., constructed with New), always has a non-nil root +// node. +func (t *Trie) Root() *Node { + return t.root +} + +// A Node represents a logical vertex in the trie structure. +type Node struct { + branches [256]*Node + terminal bool + hasChildren bool +} + +// Walk returns the node reached along edge c, if one exists. The ok value indicates whether such a node +// exist. +func (n *Node) Walk(c byte) (next *Node, ok bool) { + next = n.branches[int(c)] + return next, (next != nil) +} + +// Terminal indicates whether n is terminal in the trie (that is, whether the path from the root to n +// represents an element in the set). For instance, if the root node is terminal, then []byte{} is in the +// trie. +func (n *Node) Terminal() bool { + return n.terminal +} + +// Leaf indicates whether n is a leaf node in the trie (that is, whether it has children). A leaf node must be +// terminal (else it would not exist). Logically, if n is a leaf node then the []byte represented by the path +// from the root to n is not a proper prefix of any element of the trie. +func (n *Node) Leaf() bool { + return !n.hasChildren +} diff --git a/vendor/gopkg.in/src-d/enry.v1/.gitignore b/vendor/gopkg.in/src-d/enry.v1/.gitignore new file mode 100644 index 000000000..57b21f4e3 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/.gitignore @@ -0,0 +1,7 @@ +.linguist +benchmarks/output +.ci +Makefile.main +.shared +.idea +.docsrv-resources \ No newline at end of file diff --git a/vendor/gopkg.in/src-d/enry.v1/.travis.yml b/vendor/gopkg.in/src-d/enry.v1/.travis.yml new file mode 100644 index 000000000..781e8b3fe --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/.travis.yml @@ -0,0 +1,154 @@ +language: go + +go: + - 1.9.x + - tip + +addons: + apt: + packages: + - libonig-dev + +matrix: + allow_failures: + - go: tip + fast_finish: true + +env: + - ONIGUMURA=0 + - ONIGUMURA=1 + +install: + - rm -rf $GOPATH/src/gopkg.in/src-d + - mkdir -p $GOPATH/src/gopkg.in/src-d + - ln -s $PWD $GOPATH/src/gopkg.in/src-d/enry.v1 + - cd $GOPATH/src/gopkg.in/src-d/enry.v1 + - if [ "$ONIGUMURA" == "1" ]; then make onigumura; fi + - go get -v -t ./... + +script: + - make test-coverage + +after_success: + - bash <(curl -s https://codecov.io/bash) + +before_deploy: + - make packages + +deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file_glob: true + file: build/*.tar.gz + skip_cleanup: true + on: + tags: true + +jobs: + env: + - ONIGUMURA=0 + include: + - stage: test + language: scala + jdk: oraclejdk8 + + install: + - GIMME_OUTPUT=$(gimme 1.8 | tee -a $HOME/.bashrc) && eval "$GIMME_OUTPUT" + - export GOPATH=$HOME/gopath + - mkdir -p $GOPATH/src/gopkg.in/src-d/enry.v1 + - rsync -az ${TRAVIS_BUILD_DIR}/ $GOPATH/src/gopkg.in/src-d/enry.v1 + - go get -v gopkg.in/src-d/enry.v1/... + + before_script: + - cd java + - make + + script: + - make test + + before_deploy: + - cd .. + + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file: + - ./.shared/linux-x86-64/libenry.so + skip_cleanup: true + on: + tags: true + + - stage: Build macOS shared + + env: + - OSXCROSS_PACKAGE="osxcross_3034f7149716d815bc473d0a7b35d17e4cf175aa.tar.gz" + - OSXCROSS_URL="https://github.com/bblfsh/client-scala/releases/download/v1.5.2/${OSXCROSS_PACKAGE}" + - PATH="/$HOME/osxcross/bin:$PATH" + + sudo: true + + install: + - if [[ -z "$TRAVIS_TAG" ]]; then echo "Skipping this build for non-tag builds."; exit 0; fi + - rm -rf $GOPATH/src/gopkg.in/src-d + - mkdir -p $GOPATH/src/gopkg.in/src-d + - ln -s $PWD $GOPATH/src/gopkg.in/src-d/enry.v1 + - cd $GOPATH/src/gopkg.in/src-d/enry.v1 + - go get -v -t ./... + - sudo apt-get update + - sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils + - cd ${HOME} + - curl -sSL ${OSXCROSS_URL} | tar -C ${HOME} -xzf - + - cd $GOPATH/src/gopkg.in/src-d/enry.v1 + + script: + - make darwin-shared + + before_deploy: + - echo "skip before_deploy" + + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file: ./.shared/darwin/libenry.dylib + skip_cleanup: true + on: + tags: true + + - stage: Publish Maven + language: scala + jdk: oraclejdk8 + + before_script: + - if [[ -z "$TRAVIS_TAG" ]]; then echo "Skipping this build for non-tag builds."; exit 0; fi + - cd java + - make + - curl -o ./shared/linux-x86-64/libenry.so -sL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.so" + - mkdir -p ./shared/darwin + - curl -o ./shared/darwin/libenry.dylib -sL "https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG/libenry.dylib" + - openssl aes-256-cbc -K $encrypted_a0e1c69dbbc7_key -iv $encrypted_a0e1c69dbbc7_iv -in key.asc.enc -out key.asc -d + - gpg --no-default-keyring --primary-keyring ./project/.gnupg/pubring.gpg --secret-keyring ./project/.gnupg/secring.gpg --keyring ./project/.gnupg/pubring.gpg --fingerprint --import key.asc + + script: + - make test # ensure the shared objects are functional + - ./sbt publishLocal + - ./sbt publishSigned + - ./sbt sonatypeRelease + + before_deploy: + - rm ./target/enry-java-*-javadoc.jar + - rm ./target/enry-java-*-sources.jar + - rm ./target/enry-java-*-tests.jar + - rm ./target/enry-java-assembly-*.jar + + deploy: + provider: releases + api_key: + secure: $GITHUB_TOKEN + file_glob: true + file: ./target/enry-java*.jar + skip_cleanup: true + on: + tags: true diff --git a/vendor/gopkg.in/src-d/enry.v1/DCO b/vendor/gopkg.in/src-d/enry.v1/DCO new file mode 100644 index 000000000..29c1b9208 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/DCO @@ -0,0 +1,25 @@ + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/vendor/gopkg.in/src-d/enry.v1/LICENSE b/vendor/gopkg.in/src-d/enry.v1/LICENSE new file mode 100644 index 000000000..7f5eae069 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Sourced Technologies, S.L. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/gopkg.in/src-d/enry.v1/MAINTAINERS b/vendor/gopkg.in/src-d/enry.v1/MAINTAINERS new file mode 100644 index 000000000..94fa46981 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/MAINTAINERS @@ -0,0 +1 @@ +Alfredo Beaumont (@abeaumont) diff --git a/vendor/gopkg.in/src-d/enry.v1/Makefile b/vendor/gopkg.in/src-d/enry.v1/Makefile new file mode 100644 index 000000000..4d6eeb8df --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/Makefile @@ -0,0 +1,96 @@ +# Package configuration +PROJECT = enry +COMMANDS = cli/enry + +# Including ci Makefile +MAKEFILE = Makefile.main +CI_REPOSITORY = https://github.com/src-d/ci.git +CI_FOLDER = .ci +$(MAKEFILE): + @git clone --quiet $(CI_REPOSITORY) $(CI_FOLDER); \ + cp $(CI_FOLDER)/$(MAKEFILE) .; +-include $(MAKEFILE) + +# Docsrv: configure the languages whose api-doc can be auto generated +LANGUAGES = go +# Docs: do not edit this +DOCS_REPOSITORY := https://github.com/src-d/docs +SHARED_PATH ?= $(shell pwd)/.docsrv-resources +DOCS_PATH ?= $(SHARED_PATH)/.docs +$(DOCS_PATH)/Makefile.inc: + git clone --quiet --depth 1 $(DOCS_REPOSITORY) $(DOCS_PATH); +-include $(DOCS_PATH)/Makefile.inc + +LINGUIST_PATH = .linguist + +# build CLI +LOCAL_TAG := $(shell git describe --tags --abbrev=0) +LOCAL_COMMIT := $(shell git rev-parse --short HEAD) +LOCAL_BUILD := $(shell date +"%m-%d-%Y_%H_%M_%S") +LOCAL_LDFLAGS = -s -X main.version=$(LOCAL_TAG) -X main.build=$(LOCAL_BUILD) -X main.commit=$(LOCAL_COMMIT) + +# shared objects +RESOURCES_DIR=./.shared +LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64 +LINUX_SHARED_LIB=$(LINUX_DIR)/libenry.so +DARWIN_DIR=$(RESOURCES_DIR)/darwin +DARWIN_SHARED_LIB=$(DARWIN_DIR)/libenry.dylib +HEADER_FILE=libenry.h +NATIVE_LIB=./shared/enry.go + +# source files to be patched for using "rubex" instead of "regexp" +RUBEX_PATCHED := internal/code-generator/generator/heuristics.go internal/tokenizer/tokenize.go common.go +RUBEX_ORIG := $(RUBEX_PATCHED:=.orig) + +.PHONY: revert-onigumura + +$(LINGUIST_PATH): + git clone https://github.com/github/linguist.git $@ + +clean-linguist: + rm -rf $(LINGUIST_PATH) + +clean-shared: + rm -rf $(RESOURCES_DIR) + +clean: clean-linguist clean-shared + +code-generate: $(LINGUIST_PATH) + mkdir -p data + go run internal/code-generator/main.go + +benchmarks: $(LINGUIST_PATH) + go test -run=NONE -bench=. && benchmarks/linguist-total.sh + +benchmarks-samples: $(LINGUIST_PATH) + go test -run=NONE -bench=. -benchtime=5us && benchmarks/linguist-samples.rb + +benchmarks-slow: $(LINGUST_PATH) + mkdir -p benchmarks/output && go test -run=NONE -bench=. -slow -benchtime=100ms -timeout=100h >benchmarks/output/enry_samples.bench && \ + benchmarks/linguist-samples.rb 5 >benchmarks/output/linguist_samples.bench + +$(RUBEX_ORIG): %.orig : % + sed -i.orig -e 's/"regexp"/regexp "github.com\/moovweb\/rubex"/g' $< + @touch $@ + +onigumura: $(RUBEX_ORIG) + +revert-onigumura: + @for file in $(RUBEX_PATCHED); do if [ -e "$$file.orig" ]; then mv "$$file.orig" "$$file" && echo mv "$$file.orig" "$$file"; fi; done + +build-cli: + go build -o enry -ldflags "$(LOCAL_LDFLAGS)" cli/enry/main.go + +linux-shared: $(LINUX_SHARED_LIB) + +darwin-shared: $(DARWIN_SHARED_LIB) + +$(DARWIN_SHARED_LIB): + mkdir -p $(DARWIN_DIR) && \ + CC="o64-clang" CXX="o64-clang++" CGO_ENABLED=1 GOOS=darwin go build -buildmode=c-shared -o $(DARWIN_SHARED_LIB) $(NATIVE_LIB) && \ + mv $(DARWIN_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) + +$(LINUX_SHARED_LIB): + mkdir -p $(LINUX_DIR) && \ + GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \ + mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE) diff --git a/vendor/gopkg.in/src-d/enry.v1/README.md b/vendor/gopkg.in/src-d/enry.v1/README.md new file mode 100644 index 000000000..da6b70dbd --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/README.md @@ -0,0 +1,244 @@ +# enry [![GoDoc](https://godoc.org/gopkg.in/src-d/enry.v1?status.svg)](https://godoc.org/gopkg.in/src-d/enry.v1) [![Build Status](https://travis-ci.org/src-d/enry.svg?branch=master)](https://travis-ci.org/src-d/enry) [![codecov](https://codecov.io/gh/src-d/enry/branch/master/graph/badge.svg)](https://codecov.io/gh/src-d/enry) + +File programming language detector and toolbox to ignore binary or vendored files. *enry*, started as a port to _Go_ of the original [linguist](https://github.com/github/linguist) _Ruby_ library, that has an improved *2x performance*. + + +Installation +------------ + +The recommended way to install enry is + +``` +go get gopkg.in/src-d/enry.v1/... +``` + +To build enry's CLI you must run + + make build-cli + +this will generate a binary in the project's root directory called `enry`. You can then move this binary to anywhere in your `PATH`. + + +### Faster regexp engine (optional) + +[Onigumura](https://github.com/kkos/oniguruma) is CRuby's regular expression engine. +It is very fast and performs better than the one built into Go runtime. *enry* supports swapping +between those two engines thanks to [rubex](https://github.com/moovweb/rubex) project. +The typical overall speedup from using Onigumura is 1.5-2x. However, it requires CGo and the external shared library. +On macOS with brew, it is + +``` +brew install onigumura +``` + +On Ubuntu, it is + +``` +sudo apt install libonig-dev +``` + +To build enry with Onigumura regexps, patch the imports with + +``` +make onigumura +``` + +and then rebuild the project. + +Examples +------------ + +```go +lang, safe := enry.GetLanguageByExtension("foo.go") +fmt.Println(lang, safe) +// result: Go true + +lang, safe := enry.GetLanguageByContent("foo.m", []byte("")) +fmt.Println(lang, safe) +// result: Matlab true + +lang, safe := enry.GetLanguageByContent("bar.m", []byte("")) +fmt.Println(lang, safe) +// result: Objective-C true + +// all strategies together +lang := enry.GetLanguage("foo.cpp", []byte("")) +// result: C++ true +``` + +Note that the returned boolean value `safe` is set either to `true`, if there is only one possible language detected, or to `false` otherwise. + +To get a list of possible languages for a given file, you can use the plural version of the detecting functions. + +```go +langs := enry.GetLanguages("foo.h", []byte("")) +// result: []string{"C", "C++", "Objective-C} + +langs := enry.GetLanguagesByExtension("foo.asc", []byte(""), nil) +// result: []string{"AGS Script", "AsciiDoc", "Public Key"} + +langs := enry.GetLanguagesByFilename("Gemfile", []byte(""), []string{}) +// result: []string{"Ruby"} +``` + + +CLI +------------ + +You can use enry as a command, + +```bash +$ enry --help + enry v1.5.0 build: 10-02-2017_14_01_07 commit: 95ef0a6cf3, based on linguist commit: 37979b2 + enry, A simple (and faster) implementation of github/linguist + usage: enry + enry [-json] [-breakdown] + enry [-json] [-breakdown] + enry [-version] +``` + +and it'll return an output similar to *linguist*'s output, + +```bash +$ enry +55.56% Shell +22.22% Ruby +11.11% Gnuplot +11.11% Go +``` + +but not only the output; its flags are also the same as *linguist*'s ones, + +```bash +$ enry --breakdown +55.56% Shell +22.22% Ruby +11.11% Gnuplot +11.11% Go + +Gnuplot +plot-histogram.gp + +Ruby +linguist-samples.rb +linguist-total.rb + +Shell +parse.sh +plot-histogram.sh +run-benchmark.sh +run-slow-benchmark.sh +run.sh + +Go +parser/main.go +``` + +even the JSON flag, + +```bash +$ enry --json +{"Gnuplot":["plot-histogram.gp"],"Go":["parser/main.go"],"Ruby":["linguist-samples.rb","linguist-total.rb"],"Shell":["parse.sh","plot-histogram.sh","run-benchmark.sh","run-slow-benchmark.sh","run.sh"]} +``` + +Note that even if enry's CLI is compatible with linguist's, its main point is that **_enry doesn't need a git repository to work!_** + +Java bindings +------------ + +Generated Java binidings using a C shared library + JNI are located under [`java`](https://github.com/src-d/enry/blob/master/java) + +Development +------------ + +*enry* re-uses parts of original [linguist](https://github.com/github/linguist) to generate internal data structures. In order to update to the latest upstream and generate the necessary code you must run: + + go generate + +We update enry when changes are done in linguist's master branch on the following files: + +* [languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml) +* [heuristics.rb](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb) +* [vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) +* [documentation.yml](https://github.com/github/linguist/blob/master/lib/linguist/documentation.yml) + +Currently we don't have any procedure established to automatically detect changes in the linguist project and regenerate the code. +So we update the generated code as needed, without any specific criteria. + +If you want to update *enry* because of changes in linguist, you can run the *go +generate* command and do a pull request that only contains the changes in +generated files (those files in the subdirectory [data](https://github.com/src-d/enry/blob/master/data)). + +To run the tests, + + make test + + +Divergences from linguist +------------ + +Using [linguist/samples](https://github.com/github/linguist/tree/master/samples) +as a set for the tests, the following issues were found: + +* With [hello.ms](https://github.com/github/linguist/blob/master/samples/Unix%20Assembly/hello.ms) we can't detect the language (Unix Assembly) because we don't have a matcher in contentMatchers (content.go) for Unix Assembly. Linguist uses this [regexp](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.rb#L300) in its code, + + `elsif /(? 1000.00 && num <= 10000.00: + count[distributionIntervals[0]]++ + case num > 10000.00 && num <= 100000.00: + count[distributionIntervals[1]]++ + case num > 100000.00 && num <= 1000000.00: + count[distributionIntervals[2]]++ + case num > 1000000.00 && num <= 10000000.00: + count[distributionIntervals[3]]++ + case num > 10000000.00 && num <= 100000000.00: + count[distributionIntervals[4]]++ + } +} + +func writeCSV(CSVData [][]string, outPath string) error { + out, err := os.Create(outPath) + if err != nil { + return err + } + + w := csv.NewWriter(out) + w.WriteAll(CSVData) + + if err := w.Error(); err != nil { + return err + } + + return nil +} + +type parse func(data []byte, tool string) ([][]string, error) + +func generateCSV() { + bmFiles := []struct { + in string + out string + tool string + parse parse + }{ + {in: enryTotalBench, out: enryTotalCSV, tool: "enry", parse: parseTotal}, + {in: linguistTotalBench, out: linguistTotalCSV, tool: "linguist", parse: parseTotal}, + {in: enrySamplesBench, out: enrySamplesCSV, tool: "enry", parse: parseSamples}, + {in: linguistSamplesBench, out: linguistSamplesCSV, tool: "linguist", parse: parseSamples}, + } + + for _, bmFile := range bmFiles { + buf, err := ioutil.ReadFile(bmFile.in) + if err != nil { + log.Println(err) + continue + } + + info, err := bmFile.parse(buf, bmFile.tool) + if err != nil { + log.Println(err) + continue + } + + if err := writeCSV(info, filepath.Join(outDir, bmFile.out)); err != nil { + log.Println(err) + continue + } + } +} + +func parseTotal(data []byte, tool string) ([][]string, error) { + const totalLine = "_TOTAL" + parsedInfo := map[string][]string{} + buf := bufio.NewScanner(bytes.NewReader(data)) + for buf.Scan() { + line := buf.Text() + if strings.Contains(line, totalLine) { + split := strings.Fields(line) + row, err := getRow(split, tool) + if err != nil { + return nil, err + } + + parsedInfo[row[0]] = row + } + } + + if err := buf.Err(); err != nil { + return nil, err + } + + firstLine := []string{"function", "tool", "iterations", "ns/op"} + return prepareInfoForCSV(parsedInfo, firstLine), nil +} + +func getRow(line []string, tool string) ([]string, error) { + row := make([]string, 0, 3) + for _, function := range enryFunctions { + if strings.Contains(line[0], function) { + row = append(row, function) + break + } + } + + row = append(row, tool) + iterations := line[1] + row = append(row, iterations) + + average, err := getAverage(line) + if err != nil { + return nil, err + + } + + row = append(row, average) + return row, nil +} + +func getAverage(line []string) (string, error) { + average := line[len(line)-1] + if !strings.HasSuffix(average, ")") { + return line[2], nil + } + + totalTime := strings.Trim(average, "() ") + time, err := strconv.ParseFloat(totalTime, 64) + if err != nil { + return "", err + } + + iterations := line[1] + i, err := strconv.ParseFloat(iterations, 64) + if err != nil { + return "", err + } + + avg := (time * math.Pow10(9)) / i + return fmt.Sprintf("%d", int(avg)), nil +} + +func prepareInfoForCSV(parsedInfo map[string][]string, firstLine []string) [][]string { + info := createInfoWithFirstLine(firstLine, len(parsedInfo)) + for _, function := range enryFunctions { + info = append(info, parsedInfo[function]) + } + + return info +} + +func createInfoWithFirstLine(firstLine []string, sliceLength int) (info [][]string) { + if len(firstLine) > 0 { + info = make([][]string, 0, sliceLength+1) + info = append(info, firstLine) + } else { + info = make([][]string, 0, sliceLength) + } + + return +} + +type enryFuncs map[string][]string + +func newEnryFuncs() enryFuncs { + return enryFuncs{ + getLanguageFunc: nil, + classifyFunc: nil, + modelineFunc: nil, + filenameFunc: nil, + shebangFunc: nil, + extensionFunc: nil, + contentFunc: nil, + } +} + +func parseSamples(data []byte, tool string) ([][]string, error) { + const sampleLine = "SAMPLE_" + parsedInfo := map[string]enryFuncs{} + buf := bufio.NewScanner(bytes.NewReader(data)) + for buf.Scan() { + line := buf.Text() + if strings.Contains(line, sampleLine) { + split := strings.Fields(line) + name := getSampleName(split[0]) + if _, ok := parsedInfo[name]; !ok { + parsedInfo[name] = newEnryFuncs() + } + + row := make([]string, 0, 4) + row = append(row, name) + r, err := getRow(split, tool) + if err != nil { + return nil, err + } + + row = append(row, r...) + function := row[1] + parsedInfo[name][function] = row + } + } + + if err := buf.Err(); err != nil { + return nil, err + } + + firstLine := []string{"file", "function", "tool", "iterations", "ns/op"} + return prepareSamplesInfoForCSV(parsedInfo, firstLine), nil +} + +func getSampleName(s string) string { + start := strings.Index(s, "SAMPLE_") + len("SAMPLE_") + suffix := fmt.Sprintf("-%d", runtime.GOMAXPROCS(-1)) + name := strings.TrimSuffix(s[start:], suffix) + return name +} + +func prepareSamplesInfoForCSV(parsedInfo map[string]enryFuncs, firstLine []string) [][]string { + info := createInfoWithFirstLine(firstLine, len(parsedInfo)*len(enryFunctions)) + orderedKeys := sortKeys(parsedInfo) + for _, path := range orderedKeys { + sampleInfo := prepareInfoForCSV(parsedInfo[path], nil) + info = append(info, sampleInfo...) + } + + return info +} + +func sortKeys(parsedInfo map[string]enryFuncs) []string { + keys := make([]string, 0, len(parsedInfo)) + for key := range parsedInfo { + keys = append(keys, key) + } + + sort.Strings(keys) + return keys +} diff --git a/vendor/gopkg.in/src-d/enry.v1/benchmarks/plot-histogram.gp b/vendor/gopkg.in/src-d/enry.v1/benchmarks/plot-histogram.gp new file mode 100755 index 000000000..355fb063c --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/benchmarks/plot-histogram.gp @@ -0,0 +1,21 @@ +#!/usr/bin/env gnuplot + +set terminal png large font "arial,26" size 1920,1080 +set output 'benchmarks/histogram/distribution.png' + +set datafile separator comma +set key under + +set style data histogram +set style histogram clustered gap 1 title offset 1,1 +set style fill solid noborder +set boxwidth 0.95 +set grid y +set bmargin 12 +set autoscale +set title "Number of files per processing time" + +plot newhistogram, 'benchmarks/csv/enry-distribution.csv' using 3:xtic(1) title "enry", 'benchmarks/csv/linguist-distribution.csv' using 3 title "linguist" + +unset output + diff --git a/vendor/gopkg.in/src-d/enry.v1/benchmarks/run-benchmarks.sh b/vendor/gopkg.in/src-d/enry.v1/benchmarks/run-benchmarks.sh new file mode 100755 index 000000000..aa6e1ef23 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/benchmarks/run-benchmarks.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +mkdir -p benchmarks/output && go test -run NONE -bench=. -benchtime=120s -timeout=100h >benchmarks/output/enry_total.bench && \ +benchmarks/linguist-total.rb 5 >benchmarks/output/linguist_total.bench diff --git a/vendor/gopkg.in/src-d/enry.v1/benchmarks/run.sh b/vendor/gopkg.in/src-d/enry.v1/benchmarks/run.sh new file mode 100755 index 000000000..1ae74cbc8 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/benchmarks/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +benchmarks/run-benchmarks.sh && make benchmarks-slow && \ +benchmarks/parse.sh && benchmarks/plot-histogram.gp diff --git a/vendor/gopkg.in/src-d/enry.v1/benchmarks/soft-hard-info.txt b/vendor/gopkg.in/src-d/enry.v1/benchmarks/soft-hard-info.txt new file mode 100644 index 000000000..81992f9a7 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/benchmarks/soft-hard-info.txt @@ -0,0 +1,9 @@ +# Hardware and software used to run benchmarks + +Dell XPS 9360 +Linux 4.11.6-3-ARCH #1 SMP PREEMPT Thu Jun 22 12:21:46 CEST 2017 x86_64 +go version go1.8.3 linux/amd64 +ruby 2.4.1p111 (2017-03-22 revision 58053) [x86_64-linux] + +github/linguist/samples commit: d5c8db3fb91963c4b2762ca2ea2ff7cfac109f68 + diff --git a/vendor/gopkg.in/src-d/enry.v1/classifier.go b/vendor/gopkg.in/src-d/enry.v1/classifier.go new file mode 100644 index 000000000..8f722964c --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/classifier.go @@ -0,0 +1,102 @@ +package enry + +import ( + "math" + "sort" + + "gopkg.in/src-d/enry.v1/internal/tokenizer" +) + +// Classifier is the interface in charge to detect the possible languages of the given content based on a set of +// candidates. Candidates is a map which can be used to assign weights to languages dynamically. +type Classifier interface { + Classify(content []byte, candidates map[string]float64) (languages []string) +} + +type classifier struct { + languagesLogProbabilities map[string]float64 + tokensLogProbabilities map[string]map[string]float64 + tokensTotal float64 +} + +type scoredLanguage struct { + language string + score float64 +} + +// Classify returns a sorted slice of possible languages sorted by decreasing language's probability +func (c *classifier) Classify(content []byte, candidates map[string]float64) []string { + if len(content) == 0 { + return nil + } + + var languages map[string]float64 + if len(candidates) == 0 { + languages = c.knownLangs() + } else { + languages = make(map[string]float64, len(candidates)) + for candidate, weight := range candidates { + if lang, ok := GetLanguageByAlias(candidate); ok { + candidate = lang + } + + languages[candidate] = weight + } + } + + tokens := tokenizer.Tokenize(content) + scoredLangs := make([]*scoredLanguage, 0, len(languages)) + for language := range languages { + scoredLang := &scoredLanguage{ + language: language, + score: c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language], + } + + scoredLangs = append(scoredLangs, scoredLang) + } + + return sortLanguagesByScore(scoredLangs) +} + +func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string { + sort.Stable(byScore(scoredLangs)) + sortedLanguages := make([]string, 0, len(scoredLangs)) + for _, scoredLang := range scoredLangs { + sortedLanguages = append(sortedLanguages, scoredLang.language) + } + + return sortedLanguages +} + +func (c *classifier) knownLangs() map[string]float64 { + langs := make(map[string]float64, len(c.languagesLogProbabilities)) + for lang := range c.languagesLogProbabilities { + langs[lang]++ + } + + return langs +} + +func (c *classifier) tokensLogProbability(tokens []string, language string) float64 { + var sum float64 + for _, token := range tokens { + sum += c.tokenProbability(token, language) + } + + return sum +} + +func (c *classifier) tokenProbability(token, language string) float64 { + tokenProb, ok := c.tokensLogProbabilities[language][token] + if !ok { + tokenProb = math.Log(1.000000 / c.tokensTotal) + } + + return tokenProb +} + +type byScore []*scoredLanguage + +func (b byScore) Len() int { return len(b) } +func (b byScore) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byScore) Less(i, j int) bool { return b[j].score < b[i].score } diff --git a/vendor/gopkg.in/src-d/enry.v1/cli/enry/main.go b/vendor/gopkg.in/src-d/enry.v1/cli/enry/main.go new file mode 100644 index 000000000..177b9a02e --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/cli/enry/main.go @@ -0,0 +1,220 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/src-d/enry.v1" + "gopkg.in/src-d/enry.v1/data" +) + +var ( + version = "undefined" + build = "undefined" + commit = "undefined" +) + +func main() { + flag.Usage = usage + breakdownFlag := flag.Bool("breakdown", false, "") + jsonFlag := flag.Bool("json", false, "") + showVersion := flag.Bool("version", false, "Show the enry version information") + flag.Parse() + + if *showVersion { + fmt.Println(version) + return + } + + root, err := filepath.Abs(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + + fileInfo, err := os.Stat(root) + if err != nil { + log.Fatal(err) + } + + if fileInfo.Mode().IsRegular() { + printFileAnalysis(root) + return + } + + out := make(map[string][]string, 0) + err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + log.Println(err) + return filepath.SkipDir + } + + if !f.Mode().IsDir() && !f.Mode().IsRegular() { + return nil + } + + relativePath, err := filepath.Rel(root, path) + if err != nil { + log.Println(err) + return nil + } + + if relativePath == "." { + return nil + } + + if f.IsDir() { + relativePath = relativePath + "/" + } + + if enry.IsVendor(relativePath) || enry.IsDotFile(relativePath) || + enry.IsDocumentation(relativePath) || enry.IsConfiguration(relativePath) { + if f.IsDir() { + return filepath.SkipDir + } + + return nil + } + + if f.IsDir() { + return nil + } + + language, ok := enry.GetLanguageByExtension(path) + if !ok { + if language, ok = enry.GetLanguageByFilename(path); !ok { + content, err := ioutil.ReadFile(path) + if err != nil { + log.Println(err) + return nil + } + + language = enry.GetLanguage(filepath.Base(path), content) + if language == enry.OtherLanguage { + return nil + } + } + } + + out[language] = append(out[language], relativePath) + return nil + }) + + if err != nil { + log.Fatal(err) + } + + var buff bytes.Buffer + switch { + case *jsonFlag && !*breakdownFlag: + printJson(out, &buff) + case *jsonFlag && *breakdownFlag: + printBreakDown(out, &buff) + case *breakdownFlag: + printPercents(out, &buff) + buff.WriteByte('\n') + printBreakDown(out, &buff) + default: + printPercents(out, &buff) + } + + fmt.Print(buff.String()) +} + +func usage() { + fmt.Fprintf( + os.Stderr, + ` %[1]s %[2]s build: %[3]s commit: %[4]s, based on linguist commit: %[5]s + %[1]s, A simple (and faster) implementation of github/linguist + usage: %[1]s + %[1]s [-json] [-breakdown] + %[1]s [-json] [-breakdown] + %[1]s [-version] +`, + os.Args[0], version, build, commit, data.LinguistCommit[:7], + ) +} + +func printBreakDown(out map[string][]string, buff *bytes.Buffer) { + for name, language := range out { + writeStringLn(name, buff) + for _, file := range language { + writeStringLn(file, buff) + } + + writeStringLn("", buff) + } +} + +func printJson(out map[string][]string, buff *bytes.Buffer) { + data, _ := json.Marshal(out) + buff.Write(data) + buff.WriteByte('\n') +} + +func printPercents(out map[string][]string, buff *bytes.Buffer) { + var fileCountList enry.FileCountList + total := 0 + for name, language := range out { + fc := enry.FileCount{Name: name, Count: len(language)} + fileCountList = append(fileCountList, fc) + total += len(language) + } + // Sort the fileCountList in descending order of their count value. + sort.Sort(sort.Reverse(fileCountList)) + + for _, fc := range fileCountList { + percent := float32(fc.Count) / float32(total) * 100 + buff.WriteString(fmt.Sprintf("%.2f%% %s\n", percent, fc.Name)) + } +} + +func printFileAnalysis(file string) { + content, err := ioutil.ReadFile(file) + if err != nil { + fmt.Println(err) + } + + totalLines, nonBlank := getLines(file, string(content)) + fileType := getFileType(file, content) + language := enry.GetLanguage(file, content) + mimeType := enry.GetMimeType(file, language) + + fmt.Printf( + `%s: %d lines (%d sloc) + type: %s + mime_type: %s + language: %s +`, + filepath.Base(file), totalLines, nonBlank, fileType, mimeType, language, + ) +} + +func getLines(file string, content string) (int, int) { + totalLines := strings.Count(content, "\n") + nonBlank := totalLines - strings.Count(content, "\n\n") + return totalLines, nonBlank +} + +func getFileType(file string, content []byte) string { + switch { + case enry.IsImage(file): + return "Image" + case enry.IsBinary(content): + return "Binary" + default: + return "Text" + } +} + +func writeStringLn(s string, buff *bytes.Buffer) { + buff.WriteString(s) + buff.WriteByte('\n') +} diff --git a/vendor/gopkg.in/src-d/enry.v1/common.go b/vendor/gopkg.in/src-d/enry.v1/common.go new file mode 100644 index 000000000..9db5a69e2 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/common.go @@ -0,0 +1,463 @@ +package enry + +import ( + "bufio" + "bytes" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/src-d/enry.v1/data" +) + +// OtherLanguage is used as a zero value when a function can not return a specific language. +const OtherLanguage = "" + +// Strategy type fix the signature for the functions that can be used as a strategy. +type Strategy func(filename string, content []byte, candidates []string) (languages []string) + +// DefaultStrategies is the strategies' sequence GetLanguage uses to detect languages. +var DefaultStrategies = []Strategy{ + GetLanguagesByModeline, + GetLanguagesByFilename, + GetLanguagesByShebang, + GetLanguagesByExtension, + GetLanguagesByContent, + GetLanguagesByClassifier, +} + +var DefaultClassifier Classifier = &classifier{ + languagesLogProbabilities: data.LanguagesLogProbabilities, + tokensLogProbabilities: data.TokensLogProbabilities, + tokensTotal: data.TokensTotal, +} + +// GetLanguage applies a sequence of strategies based on the given filename and content +// to find out the most probably language to return. +func GetLanguage(filename string, content []byte) (language string) { + languages := GetLanguages(filename, content) + return firstLanguage(languages) +} + +func firstLanguage(languages []string) string { + if len(languages) == 0 { + return OtherLanguage + } + + return languages[0] +} + +// GetLanguageByModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil) +} + +// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil) +} + +// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByVimModeline(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil) +} + +// GetLanguageByFilename returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByFilename(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil) +} + +// GetLanguageByShebang returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByShebang(content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil) +} + +// GetLanguageByExtension returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByExtension(filename string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil) +} + +// GetLanguageByContent returns detected language. If there are more than one possibles languages +// it returns the first language by alphabetically order and safe to false. +func GetLanguageByContent(filename string, content []byte) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByContent, filename, content, nil) +} + +// GetLanguageByClassifier returns the most probably language detected for the given content. It uses +// DefaultClassifier, if no candidates are provided it returns OtherLanguage. +func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) { + return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates) +} + +func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) { + languages := strategy(filename, content, candidates) + return getFirstLanguageAndSafe(languages) +} + +func getFirstLanguageAndSafe(languages []string) (language string, safe bool) { + language = firstLanguage(languages) + safe = len(languages) == 1 + return +} + +// GetLanguageBySpecificClassifier returns the most probably language for the given content using +// classifier to detect language. +func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) { + languages := GetLanguagesBySpecificClassifier(content, candidates, classifier) + return getFirstLanguageAndSafe(languages) +} + +// GetLanguages applies a sequence of strategies based on the given filename and content +// to find out the most probably languages to return. +func GetLanguages(filename string, content []byte) []string { + if IsBinary(content) { + return nil + } + + var languages []string + candidates := []string{} + for _, strategy := range DefaultStrategies { + languages = strategy(filename, content, candidates) + if len(languages) == 1 { + return languages + } + + if len(languages) > 0 { + candidates = append(candidates, languages...) + } + } + + return languages +} + +// GetLanguagesByModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByModeline(_ string, content []byte, candidates []string) []string { + headFoot := getHeaderAndFooter(content) + var languages []string + for _, getLang := range modelinesFunc { + languages = getLang("", headFoot, candidates) + if len(languages) > 0 { + break + } + } + + return languages +} + +var modelinesFunc = []Strategy{ + GetLanguagesByEmacsModeline, + GetLanguagesByVimModeline, +} + +func getHeaderAndFooter(content []byte) []byte { + const searchScope = 5 + + if len(content) == 0 { + return content + } + + if bytes.Count(content, []byte("\n")) < 2*searchScope { + return content + } + + header := headScope(content, searchScope) + footer := footScope(content, searchScope) + headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:])) + headerAndFooter = append(headerAndFooter, content[:header]...) + headerAndFooter = append(headerAndFooter, content[footer:]...) + return headerAndFooter +} + +func headScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + eol := bytes.IndexAny(content, "\n") + content = content[eol+1:] + index += eol + } + + return index + scope - 1 +} + +func footScope(content []byte, scope int) (index int) { + for i := 0; i < scope; i++ { + index = bytes.LastIndexAny(content, "\n") + content = content[:index] + } + + return index + 1 +} + +var ( + reEmacsModeline = regexp.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`) + reEmacsLang = regexp.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`) + reVimModeline = regexp.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`) + reVimLang = regexp.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`) +) + +// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByEmacsModeline(_ string, content []byte, _ []string) []string { + matched := reEmacsModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched) + var alias string + if matchedAlias != nil { + alias = string(matchedAlias[1]) + } else { + alias = string(lastLineMatched) + } + + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil + } + + return []string{language} +} + +// GetLanguagesByVimModeline returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByVimModeline(_ string, content []byte, _ []string) []string { + matched := reVimModeline.FindAllSubmatch(content, -1) + if matched == nil { + return nil + } + + // only take the last matched line, discard previous lines + lastLineMatched := matched[len(matched)-1][1] + matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1) + if matchedAlias == nil { + return nil + } + + alias := string(matchedAlias[0][1]) + if len(matchedAlias) > 1 { + // cases: + // matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage; + // matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python"; + for _, match := range matchedAlias { + otherAlias := string(match[1]) + if otherAlias != alias { + return nil + } + } + } + + language, ok := GetLanguageByAlias(alias) + if !ok { + return nil + } + + return []string{language} +} + +// GetLanguagesByFilename returns a slice of possible languages for the given filename. +// It complies with the signature to be a Strategy type. +func GetLanguagesByFilename(filename string, _ []byte, _ []string) []string { + if filename == "" { + return nil + } + + return data.LanguagesByFilename[filepath.Base(filename)] +} + +// GetLanguagesByShebang returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByShebang(_ string, content []byte, _ []string) (languages []string) { + interpreter := getInterpreter(content) + return data.LanguagesByInterpreter[interpreter] +} + +var ( + shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`) + pythonVersion = regexp.MustCompile(`python\d\.\d+`) +) + +func getInterpreter(data []byte) (interpreter string) { + line := getFirstLine(data) + if !hasShebang(line) { + return "" + } + + // skip shebang + line = bytes.TrimSpace(line[2:]) + splitted := bytes.Fields(line) + if len(splitted) == 0 { + return "" + } + + if bytes.Contains(splitted[0], []byte("env")) { + if len(splitted) > 1 { + interpreter = string(splitted[1]) + } + } else { + splittedPath := bytes.Split(splitted[0], []byte{'/'}) + interpreter = string(splittedPath[len(splittedPath)-1]) + } + + if interpreter == "sh" { + interpreter = lookForMultilineExec(data) + } + + if pythonVersion.MatchString(interpreter) { + interpreter = interpreter[:strings.Index(interpreter, `.`)] + } + + return +} + +func getFirstLine(data []byte) []byte { + buf := bufio.NewScanner(bytes.NewReader(data)) + buf.Scan() + line := buf.Bytes() + if err := buf.Err(); err != nil { + return nil + } + + return line +} + +func hasShebang(line []byte) bool { + const shebang = `#!` + prefix := []byte(shebang) + return bytes.HasPrefix(line, prefix) +} + +func lookForMultilineExec(data []byte) string { + const magicNumOfLines = 5 + interpreter := "sh" + + buf := bufio.NewScanner(bytes.NewReader(data)) + for i := 0; i < magicNumOfLines && buf.Scan(); i++ { + line := buf.Bytes() + if shebangExecHack.Match(line) { + interpreter = shebangExecHack.FindStringSubmatch(string(line))[1] + break + } + } + + if err := buf.Err(); err != nil { + return interpreter + } + + return interpreter +} + +// GetLanguagesByExtension returns a slice of possible languages for the given filename. +// It complies with the signature to be a Strategy type. +func GetLanguagesByExtension(filename string, _ []byte, _ []string) []string { + if !strings.Contains(filename, ".") { + return nil + } + + filename = strings.ToLower(filename) + dots := getDotIndexes(filename) + for _, dot := range dots { + ext := filename[dot:] + languages, ok := data.LanguagesByExtension[ext] + if ok { + return languages + } + } + + return nil +} + +func getDotIndexes(filename string) []int { + dots := make([]int, 0, 2) + for i, letter := range filename { + if letter == rune('.') { + dots = append(dots, i) + } + } + + return dots +} + +// GetLanguagesByContent returns a slice of possible languages for the given content. +// It complies with the signature to be a Strategy type. +func GetLanguagesByContent(filename string, content []byte, _ []string) []string { + if filename == "" { + return nil + } + + ext := strings.ToLower(filepath.Ext(filename)) + fnMatcher, ok := data.ContentMatchers[ext] + if !ok { + return nil + } + + return fnMatcher(content) +} + +// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by +// decreasing language's probability. If there are not candidates it returns nil. It complies with the signature to be a Strategy type. +func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) { + if len(candidates) == 0 { + return nil + } + + return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier) +} + +// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used. +func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) { + mapCandidates := make(map[string]float64) + for _, candidate := range candidates { + mapCandidates[candidate]++ + } + + return classifier.Classify(content, mapCandidates) +} + +// GetLanguageExtensions returns the different extensions being used by the language. +func GetLanguageExtensions(language string) []string { + return data.ExtensionsByLanguage[language] +} + +// Type represent language's type. Either data, programming, markup, prose, or unknown. +type Type int + +// Type's values. +const ( + Unknown Type = iota + Data + Programming + Markup + Prose +) + +// GetLanguageType returns the type of the given language. +func GetLanguageType(language string) (langType Type) { + intType, ok := data.LanguagesType[language] + langType = Type(intType) + if !ok { + langType = Unknown + } + return langType +} + +// GetLanguageByAlias returns either the language related to the given alias and ok set to true +// or Otherlanguage and ok set to false if the alias is not recognized. +func GetLanguageByAlias(alias string) (lang string, ok bool) { + a := strings.Split(alias, `,`)[0] + a = strings.ToLower(a) + lang, ok = data.LanguagesByAlias[a] + if !ok { + lang = OtherLanguage + } + + return +} diff --git a/vendor/gopkg.in/src-d/enry.v1/common_test.go b/vendor/gopkg.in/src-d/enry.v1/common_test.go new file mode 100644 index 000000000..b2a5f153c --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/common_test.go @@ -0,0 +1,447 @@ +package enry + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "gopkg.in/src-d/enry.v1/data" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +const linguistURL = "https://github.com/github/linguist.git" +const linguistClonedEnvVar = "ENRY_TEST_REPO" + +type EnryTestSuite struct { + suite.Suite + repoLinguist string + samplesDir string + cloned bool +} + +func TestEnryTestSuite(t *testing.T) { + suite.Run(t, new(EnryTestSuite)) +} + +func (s *EnryTestSuite) SetupSuite() { + var err error + s.repoLinguist = os.Getenv(linguistClonedEnvVar) + s.cloned = s.repoLinguist == "" + if s.cloned { + s.repoLinguist, err = ioutil.TempDir("", "linguist-") + assert.NoError(s.T(), err) + } + + s.samplesDir = filepath.Join(s.repoLinguist, "samples") + + if s.cloned { + cmd := exec.Command("git", "clone", linguistURL, s.repoLinguist) + err = cmd.Run() + assert.NoError(s.T(), err) + } + + cwd, err := os.Getwd() + assert.NoError(s.T(), err) + + err = os.Chdir(s.repoLinguist) + assert.NoError(s.T(), err) + + cmd := exec.Command("git", "checkout", data.LinguistCommit) + err = cmd.Run() + assert.NoError(s.T(), err) + + err = os.Chdir(cwd) + assert.NoError(s.T(), err) +} + +func (s *EnryTestSuite) TearDownSuite() { + if s.cloned { + err := os.RemoveAll(s.repoLinguist) + assert.NoError(s.T(), err) + } +} + +func (s *EnryTestSuite) TestGetLanguage() { + tests := []struct { + name string + filename string + content []byte + expected string + safe bool + }{ + {name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"}, + {name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"}, + {name: "TestGetLanguage_3", filename: "foo.m", content: nil, expected: OtherLanguage}, + {name: "TestGetLanguage_4", filename: "foo.mo", content: []byte{0xDE, 0x12, 0x04, 0x95, 0x00, 0x00, 0x00, 0x00}, expected: OtherLanguage}, + {name: "TestGetLanguage_5", filename: "", content: nil, expected: OtherLanguage}, + } + + for _, test := range tests { + language := GetLanguage(test.filename, test.content) + assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: %v, expected: %v", test.name, language, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByModelineLinguist() { + var modelinesDir = filepath.Join(s.repoLinguist, "test/fixtures/Data/Modelines") + + tests := []struct { + name string + filename string + candidates []string + expected []string + }{ + // Emacs + {name: "TestGetLanguagesByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expected: []string{"Smalltalk"}}, + {name: "TestGetLanguagesByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expected: []string{"Text"}}, + {name: "TestGetLanguagesByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expected: []string{"PHP"}}, + {name: "TestGetLanguagesByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expected: []string{"C++"}}, + + // Vim + {name: "TestGetLanguagesByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expected: []string{"C++"}}, + {name: "TestGetLanguagesByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expected: []string{"JavaScript"}}, + {name: "TestGetLanguagesByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expected: []string{"JavaScript"}}, + {name: "TestGetLanguagesByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expected: []string{"Prolog"}}, + {name: "TestGetLanguagesByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByModelineLinguist_31", filename: filepath.Join(s.samplesDir, "C/main.c"), expected: nil}, + {name: "TestGetLanguagesByModelineLinguist_32", filename: "", expected: nil}, + } + + for _, test := range tests { + var content []byte + var err error + + if test.filename != "" { + content, err = ioutil.ReadFile(test.filename) + assert.NoError(s.T(), err) + } + + languages := GetLanguagesByModeline(test.filename, content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByModeline() { + const ( + wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :` + rightVim = `/* vim: set syntax=python ft =python filetype=python */` + noLangVim = `/* vim: set shiftwidth=4 softtabstop=0 cindent cinoptions={1s: */` + ) + + tests := []struct { + name string + filename string + content []byte + candidates []string + expected []string + }{ + {name: "TestGetLanguagesByModeline_1", content: []byte(wrongVim), expected: nil}, + {name: "TestGetLanguagesByModeline_2", content: []byte(rightVim), expected: []string{"Python"}}, + {name: "TestGetLanguagesByModeline_3", content: []byte(noLangVim), expected: nil}, + {name: "TestGetLanguagesByModeline_4", content: nil, expected: nil}, + {name: "TestGetLanguagesByModeline_5", content: []byte{}, expected: nil}, + } + + for _, test := range tests { + languages := GetLanguagesByModeline(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByFilename() { + tests := []struct { + name string + filename string + content []byte + candidates []string + expected []string + }{ + {name: "TestGetLanguagesByFilename_1", filename: "unknown.interpreter", expected: nil}, + {name: "TestGetLanguagesByFilename_2", filename: ".bashrc", expected: []string{"Shell"}}, + {name: "TestGetLanguagesByFilename_3", filename: "Dockerfile", expected: []string{"Dockerfile"}}, + {name: "TestGetLanguagesByFilename_4", filename: "Makefile.frag", expected: []string{"Makefile"}}, + {name: "TestGetLanguagesByFilename_5", filename: "makefile", expected: []string{"Makefile"}}, + {name: "TestGetLanguagesByFilename_6", filename: "Vagrantfile", expected: []string{"Ruby"}}, + {name: "TestGetLanguagesByFilename_7", filename: "_vimrc", expected: []string{"Vim script"}}, + {name: "TestGetLanguagesByFilename_8", filename: "pom.xml", expected: []string{"Maven POM"}}, + {name: "TestGetLanguagesByFilename_9", filename: "", expected: nil}, + } + + for _, test := range tests { + languages := GetLanguagesByFilename(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByShebang() { + const ( + multilineExecHack = `#!/bin/sh +# Next line is comment in Tcl, but not in sh... \ +exec tclsh "$0" ${1+"$@"}` + + multilineNoExecHack = `#!/bin/sh +#<<<# +echo "A shell script in a zkl program ($0)" +echo "Now run zkl with Hello World as args" +zkl $0 Hello World! +exit +#<<<# +println("The shell script says ",vm.arglist.concat(" "));` + ) + + tests := []struct { + name string + filename string + content []byte + candidates []string + expected []string + }{ + {name: "TestGetLanguagesByShebang_1", content: []byte(`#!/unknown/interpreter`), expected: nil}, + {name: "TestGetLanguagesByShebang_2", content: []byte(`no shebang`), expected: nil}, + {name: "TestGetLanguagesByShebang_3", content: []byte(`#!/usr/bin/env`), expected: nil}, + {name: "TestGetLanguagesByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expected: []string{"Python"}}, + {name: "TestGetLanguagesByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expected: []string{"Python"}}, + {name: "TestGetLanguagesByShebang_6", content: []byte(`#!/usr/bin/env perl`), expected: []string{"Perl", "Pod"}}, + {name: "TestGetLanguagesByShebang_7", content: []byte(`#! /bin/sh`), expected: []string{"Shell"}}, + {name: "TestGetLanguagesByShebang_8", content: []byte(`#!bash`), expected: []string{"Shell"}}, + {name: "TestGetLanguagesByShebang_9", content: []byte(multilineExecHack), expected: []string{"Tcl"}}, + {name: "TestGetLanguagesByShebang_10", content: []byte(multilineNoExecHack), expected: []string{"Shell"}}, + {name: "TestGetLanguagesByShebang_11", content: []byte(`#!`), expected: nil}, + } + + for _, test := range tests { + languages := GetLanguagesByShebang(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByExtension() { + tests := []struct { + name string + filename string + content []byte + candidates []string + expected []string + }{ + {name: "TestGetLanguagesByExtension_1", filename: "foo.foo", expected: nil}, + {name: "TestGetLanguagesByExtension_2", filename: "foo.go", expected: []string{"Go"}}, + {name: "TestGetLanguagesByExtension_3", filename: "foo.go.php", expected: []string{"Hack", "PHP"}}, + {name: "TestGetLanguagesByExtension_4", filename: "", expected: nil}, + } + + for _, test := range tests { + languages := GetLanguagesByExtension(test.filename, test.content, test.candidates) + assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesByClassifier() { + test := []struct { + name string + filename string + candidates []string + expected string + }{ + {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "C"}, + {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: nil, expected: OtherLanguage}, + {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(s.samplesDir, "C/main.c"), candidates: []string{}, expected: OtherLanguage}, + {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expected: "C++"}, + {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, expected: "Ruby"}, + {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "Python"}, + {name: "TestGetLanguagesByClassifier_7", filename: "", candidates: []string{"python"}, expected: OtherLanguage}, + } + + for _, test := range test { + var content []byte + var err error + + if test.filename != "" { + content, err = ioutil.ReadFile(test.filename) + assert.NoError(s.T(), err) + } + + languages := GetLanguagesByClassifier(test.filename, content, test.candidates) + var language string + if len(languages) == 0 { + language = OtherLanguage + } else { + language = languages[0] + } + + assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguagesBySpecificClassifier() { + test := []struct { + name string + filename string + candidates []string + classifier Classifier + expected string + }{ + {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: nil, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(s.samplesDir, "C/main.c"), candidates: []string{}, classifier: DefaultClassifier, expected: "C"}, + {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, classifier: DefaultClassifier, expected: "C++"}, + {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(s.samplesDir, "C/blob.c"), candidates: []string{"ruby"}, classifier: DefaultClassifier, expected: "Ruby"}, + {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(s.samplesDir, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "Python"}, + {name: "TestGetLanguagesByClassifier_7", filename: os.DevNull, candidates: nil, classifier: DefaultClassifier, expected: OtherLanguage}, + } + + for _, test := range test { + content, err := ioutil.ReadFile(test.filename) + assert.NoError(s.T(), err) + + languages := GetLanguagesBySpecificClassifier(content, test.candidates, test.classifier) + var language string + if len(languages) == 0 { + language = OtherLanguage + } else { + language = languages[0] + } + + assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguageExtensions() { + tests := []struct { + name string + language string + expected []string + }{ + {name: "TestGetLanguageExtensions_1", language: "foo", expected: nil}, + {name: "TestGetLanguageExtensions_2", language: "COBOL", expected: []string{".cob", ".cbl", ".ccp", ".cobol", ".cpy"}}, + {name: "TestGetLanguageExtensions_3", language: "Maven POM", expected: nil}, + } + + for _, test := range tests { + extensions := GetLanguageExtensions(test.language) + assert.EqualValues(s.T(), test.expected, extensions, fmt.Sprintf("%v: extensions = %v, expected: %v", test.name, extensions, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguageType() { + tests := []struct { + name string + language string + expected Type + }{ + {name: "TestGetLanguageType_1", language: "BestLanguageEver", expected: Unknown}, + {name: "TestGetLanguageType_2", language: "JSON", expected: Data}, + {name: "TestGetLanguageType_3", language: "COLLADA", expected: Data}, + {name: "TestGetLanguageType_4", language: "Go", expected: Programming}, + {name: "TestGetLanguageType_5", language: "Brainfuck", expected: Programming}, + {name: "TestGetLanguageType_6", language: "HTML", expected: Markup}, + {name: "TestGetLanguageType_7", language: "Sass", expected: Markup}, + {name: "TestGetLanguageType_8", language: "AsciiDoc", expected: Prose}, + {name: "TestGetLanguageType_9", language: "Textile", expected: Prose}, + } + + for _, test := range tests { + langType := GetLanguageType(test.language) + assert.Equal(s.T(), test.expected, langType, fmt.Sprintf("%v: langType = %v, expected: %v", test.name, langType, test.expected)) + } +} + +func (s *EnryTestSuite) TestGetLanguageByAlias() { + tests := []struct { + name string + alias string + expectedLang string + expectedOk bool + }{ + {name: "TestGetLanguageByAlias_1", alias: "BestLanguageEver", expectedLang: OtherLanguage, expectedOk: false}, + {name: "TestGetLanguageByAlias_2", alias: "aspx-vb", expectedLang: "ASP", expectedOk: true}, + {name: "TestGetLanguageByAlias_3", alias: "C++", expectedLang: "C++", expectedOk: true}, + {name: "TestGetLanguageByAlias_4", alias: "c++", expectedLang: "C++", expectedOk: true}, + {name: "TestGetLanguageByAlias_5", alias: "objc", expectedLang: "Objective-C", expectedOk: true}, + {name: "TestGetLanguageByAlias_6", alias: "golang", expectedLang: "Go", expectedOk: true}, + {name: "TestGetLanguageByAlias_7", alias: "GOLANG", expectedLang: "Go", expectedOk: true}, + {name: "TestGetLanguageByAlias_8", alias: "bsdmake", expectedLang: "Makefile", expectedOk: true}, + {name: "TestGetLanguageByAlias_9", alias: "xhTmL", expectedLang: "HTML", expectedOk: true}, + {name: "TestGetLanguageByAlias_10", alias: "python", expectedLang: "Python", expectedOk: true}, + } + + for _, test := range tests { + lang, ok := GetLanguageByAlias(test.alias) + assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang)) + assert.Equal(s.T(), test.expectedOk, ok, fmt.Sprintf("%v: ok = %v, expected: %v", test.name, ok, test.expectedOk)) + } +} + +func (s *EnryTestSuite) TestLinguistCorpus() { + const filenamesDir = "filenames" + var cornerCases = map[string]bool{ + "hello.ms": true, + } + + var total, failed, ok, other int + var expected string + filepath.Walk(s.samplesDir, func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + if f.Name() != filenamesDir { + expected = f.Name() + } + + return nil + } + + filename := filepath.Base(path) + content, _ := ioutil.ReadFile(path) + + total++ + obtained := GetLanguage(filename, content) + if obtained == OtherLanguage { + obtained = "Other" + other++ + } + + var status string + if expected == obtained { + status = "ok" + ok++ + } else { + status = "failed" + failed++ + + } + + if _, ok := cornerCases[filename]; ok { + fmt.Printf("\t\t[considered corner case] %s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status) + } else { + assert.Equal(s.T(), expected, obtained, fmt.Sprintf("%s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status)) + } + + return nil + }) + + fmt.Printf("\t\ttotal files: %d, ok: %d, failed: %d, other: %d\n", total, ok, failed, other) +} diff --git a/vendor/gopkg.in/src-d/enry.v1/data/alias.go b/vendor/gopkg.in/src-d/enry.v1/data/alias.go new file mode 100644 index 000000000..c29707273 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/data/alias.go @@ -0,0 +1,664 @@ +package data + +// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/enry.v1/internal/code-generator +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: 4cd558c37482e8d2c535d8107f2d11b49afbc5b5 + +// LanguagesByAlias keeps alias for different languages and use the name of the languages as an alias too. +// All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores. +var LanguagesByAlias = map[string]string{ + "1c_enterprise": "1C Enterprise", + "abap": "ABAP", + "abl": "OpenEdge ABL", + "abnf": "ABNF", + "abuild": "Alpine Abuild", + "acfm": "Adobe Font Metrics", + "aconf": "ApacheConf", + "actionscript": "ActionScript", + "actionscript3": "ActionScript", + "actionscript_3": "ActionScript", + "ada": "Ada", + "ada2005": "Ada", + "ada95": "Ada", + "adobe_composite_font_metrics": "Adobe Font Metrics", + "adobe_font_metrics": "Adobe Font Metrics", + "adobe_multiple_font_metrics": "Adobe Font Metrics", + "advpl": "xBase", + "afdko": "OpenType Feature File", + "agda": "Agda", + "ags": "AGS Script", + "ags_script": "AGS Script", + "ahk": "AutoHotkey", + "alloy": "Alloy", + "alpine_abuild": "Alpine Abuild", + "amfm": "Adobe Font Metrics", + "ampl": "AMPL", + "ant_build_system": "Ant Build System", + "antlr": "ANTLR", + "apache": "ApacheConf", + "apacheconf": "ApacheConf", + "apex": "Apex", + "api_blueprint": "API Blueprint", + "apkbuild": "Alpine Abuild", + "apl": "APL", + "apollo_guidance_computer": "Apollo Guidance Computer", + "applescript": "AppleScript", + "arc": "Arc", + "arduino": "Arduino", + "arexx": "REXX", + "as3": "ActionScript", + "asciidoc": "AsciiDoc", + "asn.1": "ASN.1", + "asp": "ASP", + "aspectj": "AspectJ", + "aspx": "ASP", + "aspx-vb": "ASP", + "assembly": "Assembly", + "ats": "ATS", + "ats2": "ATS", + "au3": "AutoIt", + "augeas": "Augeas", + "autoconf": "M4Sugar", + "autohotkey": "AutoHotkey", + "autoit": "AutoIt", + "autoit3": "AutoIt", + "autoitscript": "AutoIt", + "awk": "Awk", + "b3d": "BlitzBasic", + "ballerina": "Ballerina", + "bash": "Shell", + "bash_session": "ShellSession", + "bat": "Batchfile", + "batch": "Batchfile", + "batchfile": "Batchfile", + "befunge": "Befunge", + "bison": "Bison", + "bitbake": "BitBake", + "blade": "Blade", + "blitz3d": "BlitzBasic", + "blitzbasic": "BlitzBasic", + "blitzmax": "BlitzMax", + "blitzplus": "BlitzBasic", + "bluespec": "Bluespec", + "bmax": "BlitzMax", + "boo": "Boo", + "bplus": "BlitzBasic", + "brainfuck": "Brainfuck", + "brightscript": "Brightscript", + "bro": "Bro", + "bsdmake": "Makefile", + "byond": "DM", + "c": "C", + "c#": "C#", + "c++": "C++", + "c++-objdump": "Cpp-ObjDump", + "c-objdump": "C-ObjDump", + "c2hs": "C2hs Haskell", + "c2hs_haskell": "C2hs Haskell", + "cap'n_proto": "Cap'n Proto", + "carto": "CartoCSS", + "cartocss": "CartoCSS", + "ceylon": "Ceylon", + "cfc": "ColdFusion CFC", + "cfm": "ColdFusion", + "cfml": "ColdFusion", + "chapel": "Chapel", + "charity": "Charity", + "chpl": "Chapel", + "chuck": "ChucK", + "cirru": "Cirru", + "clarion": "Clarion", + "clean": "Clean", + "click": "Click", + "clipper": "xBase", + "clips": "CLIPS", + "clojure": "Clojure", + "closure_templates": "Closure Templates", + "cmake": "CMake", + "cobol": "COBOL", + "coffee": "CoffeeScript", + "coffee-script": "CoffeeScript", + "coffeescript": "CoffeeScript", + "coldfusion": "ColdFusion", + "coldfusion_cfc": "ColdFusion CFC", + "coldfusion_html": "ColdFusion", + "collada": "COLLADA", + "common_lisp": "Common Lisp", + "component_pascal": "Component Pascal", + "console": "ShellSession", + "cool": "Cool", + "coq": "Coq", + "cpp": "C++", + "cpp-objdump": "Cpp-ObjDump", + "creole": "Creole", + "crystal": "Crystal", + "csharp": "C#", + "cson": "CSON", + "csound": "Csound", + "csound-csd": "Csound Document", + "csound-orc": "Csound", + "csound-sco": "Csound Score", + "csound_document": "Csound Document", + "csound_score": "Csound Score", + "css": "CSS", + "csv": "CSV", + "cucumber": "Gherkin", + "cuda": "Cuda", + "cweb": "CWeb", + "cycript": "Cycript", + "cython": "Cython", + "d": "D", + "d-objdump": "D-ObjDump", + "darcs_patch": "Darcs Patch", + "dart": "Dart", + "dataweave": "DataWeave", + "dcl": "DIGITAL Command Language", + "delphi": "Component Pascal", + "desktop": "desktop", + "diff": "Diff", + "digital_command_language": "DIGITAL Command Language", + "django": "HTML+Django", + "dm": "DM", + "dns_zone": "DNS Zone", + "dockerfile": "Dockerfile", + "dogescript": "Dogescript", + "dosbatch": "Batchfile", + "dosini": "INI", + "dpatch": "Darcs Patch", + "dtrace": "DTrace", + "dtrace-script": "DTrace", + "dylan": "Dylan", + "e": "E", + "eagle": "Eagle", + "easybuild": "Easybuild", + "ebnf": "EBNF", + "ec": "eC", + "ecere_projects": "Ecere Projects", + "ecl": "ECL", + "eclipse": "ECLiPSe", + "ecr": "HTML+ECR", + "edn": "edn", + "eeschema_schematic": "KiCad Schematic", + "eex": "HTML+EEX", + "eiffel": "Eiffel", + "ejs": "EJS", + "elisp": "Emacs Lisp", + "elixir": "Elixir", + "elm": "Elm", + "emacs": "Emacs Lisp", + "emacs_lisp": "Emacs Lisp", + "emberscript": "EmberScript", + "eq": "EQ", + "erb": "HTML+ERB", + "erlang": "Erlang", + "f#": "F#", + "factor": "Factor", + "fancy": "Fancy", + "fantom": "Fantom", + "filebench_wml": "Filebench WML", + "filterscript": "Filterscript", + "fish": "fish", + "flex": "Lex", + "flux": "FLUX", + "formatted": "Formatted", + "forth": "Forth", + "fortran": "Fortran", + "foxpro": "xBase", + "freemarker": "FreeMarker", + "frege": "Frege", + "fsharp": "F#", + "ftl": "FreeMarker", + "fundamental": "Text", + "g-code": "G-code", + "game_maker_language": "Game Maker Language", + "gams": "GAMS", + "gap": "GAP", + "gcc_machine_description": "GCC Machine Description", + "gdb": "GDB", + "gdscript": "GDScript", + "genie": "Genie", + "genshi": "Genshi", + "gentoo_ebuild": "Gentoo Ebuild", + "gentoo_eclass": "Gentoo Eclass", + "gerber_image": "Gerber Image", + "gettext_catalog": "Gettext Catalog", + "gf": "Grammatical Framework", + "gherkin": "Gherkin", + "glsl": "GLSL", + "glyph": "Glyph", + "gn": "GN", + "gnuplot": "Gnuplot", + "go": "Go", + "golang": "Go", + "golo": "Golo", + "gosu": "Gosu", + "grace": "Grace", + "gradle": "Gradle", + "grammatical_framework": "Grammatical Framework", + "graph_modeling_language": "Graph Modeling Language", + "graphql": "GraphQL", + "graphviz_(dot)": "Graphviz (DOT)", + "groovy": "Groovy", + "groovy_server_pages": "Groovy Server Pages", + "gsp": "Groovy Server Pages", + "hack": "Hack", + "haml": "Haml", + "handlebars": "Handlebars", + "harbour": "Harbour", + "haskell": "Haskell", + "haxe": "Haxe", + "hbs": "Handlebars", + "hcl": "HCL", + "hlsl": "HLSL", + "html": "HTML", + "html+django": "HTML+Django", + "html+django/jinja": "HTML+Django", + "html+ecr": "HTML+ECR", + "html+eex": "HTML+EEX", + "html+erb": "HTML+ERB", + "html+jinja": "HTML+Django", + "html+php": "HTML+PHP", + "html+ruby": "RHTML", + "htmlbars": "Handlebars", + "htmldjango": "HTML+Django", + "http": "HTTP", + "hy": "Hy", + "hylang": "Hy", + "hyphy": "HyPhy", + "i7": "Inform 7", + "idl": "IDL", + "idris": "Idris", + "igor": "IGOR Pro", + "igor_pro": "IGOR Pro", + "igorpro": "IGOR Pro", + "inc": "PHP", + "inform7": "Inform 7", + "inform_7": "Inform 7", + "ini": "INI", + "inno_setup": "Inno Setup", + "io": "Io", + "ioke": "Ioke", + "ipython_notebook": "Jupyter Notebook", + "irc": "IRC log", + "irc_log": "IRC log", + "irc_logs": "IRC log", + "isabelle": "Isabelle", + "isabelle_root": "Isabelle ROOT", + "j": "J", + "jasmin": "Jasmin", + "java": "Java", + "java_server_page": "Groovy Server Pages", + "java_server_pages": "Java Server Pages", + "javascript": "JavaScript", + "jflex": "JFlex", + "jison": "Jison", + "jison_lex": "Jison Lex", + "jolie": "Jolie", + "jruby": "Ruby", + "js": "JavaScript", + "json": "JSON", + "json5": "JSON5", + "jsoniq": "JSONiq", + "jsonld": "JSONLD", + "jsp": "Java Server Pages", + "jsx": "JSX", + "julia": "Julia", + "jupyter_notebook": "Jupyter Notebook", + "kicad_layout": "KiCad Layout", + "kicad_legacy_layout": "KiCad Legacy Layout", + "kicad_schematic": "KiCad Schematic", + "kit": "Kit", + "kotlin": "Kotlin", + "krl": "KRL", + "labview": "LabVIEW", + "lasso": "Lasso", + "lassoscript": "Lasso", + "latex": "TeX", + "latte": "Latte", + "lean": "Lean", + "less": "Less", + "lex": "Lex", + "lfe": "LFE", + "lhaskell": "Literate Haskell", + "lhs": "Literate Haskell", + "lilypond": "LilyPond", + "limbo": "Limbo", + "linker_script": "Linker Script", + "linux_kernel_module": "Linux Kernel Module", + "liquid": "Liquid", + "lisp": "Common Lisp", + "litcoffee": "Literate CoffeeScript", + "literate_agda": "Literate Agda", + "literate_coffeescript": "Literate CoffeeScript", + "literate_haskell": "Literate Haskell", + "live-script": "LiveScript", + "livescript": "LiveScript", + "llvm": "LLVM", + "logos": "Logos", + "logtalk": "Logtalk", + "lolcode": "LOLCODE", + "lookml": "LookML", + "loomscript": "LoomScript", + "ls": "LiveScript", + "lsl": "LSL", + "lua": "Lua", + "m": "M", + "m4": "M4", + "m4sugar": "M4Sugar", + "macruby": "Ruby", + "make": "Makefile", + "makefile": "Makefile", + "mako": "Mako", + "markdown": "Markdown", + "marko": "Marko", + "markojs": "Marko", + "mask": "Mask", + "mathematica": "Mathematica", + "matlab": "Matlab", + "maven_pom": "Maven POM", + "max": "Max", + "max/msp": "Max", + "maxmsp": "Max", + "maxscript": "MAXScript", + "mediawiki": "MediaWiki", + "mercury": "Mercury", + "meson": "Meson", + "metal": "Metal", + "mf": "Makefile", + "minid": "MiniD", + "mirah": "Mirah", + "mma": "Mathematica", + "modelica": "Modelica", + "modula-2": "Modula-2", + "module_management_system": "Module Management System", + "monkey": "Monkey", + "moocode": "Moocode", + "moonscript": "MoonScript", + "mql4": "MQL4", + "mql5": "MQL5", + "mtml": "MTML", + "muf": "MUF", + "mumps": "M", + "mupad": "mupad", + "myghty": "Myghty", + "nasm": "Assembly", + "ncl": "NCL", + "nearley": "Nearley", + "nemerle": "Nemerle", + "nesc": "nesC", + "netlinx": "NetLinx", + "netlinx+erb": "NetLinx+ERB", + "netlogo": "NetLogo", + "newlisp": "NewLisp", + "nginx": "Nginx", + "nginx_configuration_file": "Nginx", + "nim": "Nim", + "ninja": "Ninja", + "nit": "Nit", + "nix": "Nix", + "nixos": "Nix", + "njk": "HTML+Django", + "nl": "NL", + "node": "JavaScript", + "nroff": "Roff", + "nsis": "NSIS", + "nu": "Nu", + "numpy": "NumPy", + "nunjucks": "HTML+Django", + "nush": "Nu", + "nvim": "Vim script", + "obj-c": "Objective-C", + "obj-c++": "Objective-C++", + "obj-j": "Objective-J", + "objc": "Objective-C", + "objc++": "Objective-C++", + "objdump": "ObjDump", + "objective-c": "Objective-C", + "objective-c++": "Objective-C++", + "objective-j": "Objective-J", + "objectivec": "Objective-C", + "objectivec++": "Objective-C++", + "objectivej": "Objective-J", + "objectpascal": "Component Pascal", + "objj": "Objective-J", + "ocaml": "OCaml", + "octave": "Matlab", + "omgrofl": "Omgrofl", + "ooc": "ooc", + "opa": "Opa", + "opal": "Opal", + "opencl": "OpenCL", + "openedge": "OpenEdge ABL", + "openedge_abl": "OpenEdge ABL", + "openrc": "OpenRC runscript", + "openrc_runscript": "OpenRC runscript", + "openscad": "OpenSCAD", + "opentype_feature_file": "OpenType Feature File", + "org": "Org", + "osascript": "AppleScript", + "ox": "Ox", + "oxygene": "Oxygene", + "oz": "Oz", + "p4": "P4", + "pan": "Pan", + "pandoc": "Markdown", + "papyrus": "Papyrus", + "parrot": "Parrot", + "parrot_assembly": "Parrot Assembly", + "parrot_internal_representation": "Parrot Internal Representation", + "pascal": "Pascal", + "pasm": "Parrot Assembly", + "pawn": "PAWN", + "pcbnew": "KiCad Layout", + "pep8": "Pep8", + "perl": "Perl", + "perl_6": "Perl 6", + "php": "PHP", + "pic": "Pic", + "pickle": "Pickle", + "picolisp": "PicoLisp", + "piglatin": "PigLatin", + "pike": "Pike", + "pir": "Parrot Internal Representation", + "plpgsql": "PLpgSQL", + "plsql": "PLSQL", + "pod": "Pod", + "pogoscript": "PogoScript", + "pony": "Pony", + "posh": "PowerShell", + "postscr": "PostScript", + "postscript": "PostScript", + "pot": "Gettext Catalog", + "pov-ray": "POV-Ray SDL", + "pov-ray_sdl": "POV-Ray SDL", + "povray": "POV-Ray SDL", + "powerbuilder": "PowerBuilder", + "powershell": "PowerShell", + "processing": "Processing", + "progress": "OpenEdge ABL", + "prolog": "Prolog", + "propeller_spin": "Propeller Spin", + "protobuf": "Protocol Buffer", + "protocol_buffer": "Protocol Buffer", + "protocol_buffers": "Protocol Buffer", + "public_key": "Public Key", + "pug": "Pug", + "puppet": "Puppet", + "pure_data": "Pure Data", + "purebasic": "PureBasic", + "purescript": "PureScript", + "pycon": "Python console", + "pyrex": "Cython", + "python": "Python", + "python_console": "Python console", + "python_traceback": "Python traceback", + "qmake": "QMake", + "qml": "QML", + "r": "R", + "racket": "Racket", + "ragel": "Ragel", + "ragel-rb": "Ragel", + "ragel-ruby": "Ragel", + "rake": "Ruby", + "raml": "RAML", + "rascal": "Rascal", + "raw": "Raw token data", + "raw_token_data": "Raw token data", + "rb": "Ruby", + "rbx": "Ruby", + "rdoc": "RDoc", + "realbasic": "REALbasic", + "reason": "Reason", + "rebol": "Rebol", + "red": "Red", + "red/system": "Red", + "redcode": "Redcode", + "regex": "Regular Expression", + "regexp": "Regular Expression", + "regular_expression": "Regular Expression", + "ren'py": "Ren'Py", + "renderscript": "RenderScript", + "renpy": "Ren'Py", + "restructuredtext": "reStructuredText", + "rexx": "REXX", + "rhtml": "RHTML", + "ring": "Ring", + "rmarkdown": "RMarkdown", + "robotframework": "RobotFramework", + "roff": "Roff", + "rouge": "Rouge", + "rpm_spec": "RPM Spec", + "rs-274x": "Gerber Image", + "rscript": "R", + "rss": "XML", + "rst": "reStructuredText", + "ruby": "Ruby", + "runoff": "RUNOFF", + "rust": "Rust", + "rusthon": "Python", + "sage": "Sage", + "salt": "SaltStack", + "saltstack": "SaltStack", + "saltstate": "SaltStack", + "sas": "SAS", + "sass": "Sass", + "scala": "Scala", + "scaml": "Scaml", + "scheme": "Scheme", + "scilab": "Scilab", + "scss": "SCSS", + "self": "Self", + "sh": "Shell", + "shaderlab": "ShaderLab", + "shell": "Shell", + "shell-script": "Shell", + "shellsession": "ShellSession", + "shen": "Shen", + "slash": "Slash", + "slim": "Slim", + "smali": "Smali", + "smalltalk": "Smalltalk", + "smarty": "Smarty", + "sml": "Standard ML", + "smt": "SMT", + "sourcemod": "SourcePawn", + "sourcepawn": "SourcePawn", + "sparql": "SPARQL", + "specfile": "RPM Spec", + "spline_font_database": "Spline Font Database", + "splus": "R", + "sqf": "SQF", + "sql": "SQL", + "sqlpl": "SQLPL", + "squeak": "Smalltalk", + "squirrel": "Squirrel", + "srecode_template": "SRecode Template", + "stan": "Stan", + "standard_ml": "Standard ML", + "stata": "Stata", + "ston": "STON", + "stylus": "Stylus", + "sublime_text_config": "Sublime Text Config", + "subrip_text": "SubRip Text", + "supercollider": "SuperCollider", + "svg": "SVG", + "swift": "Swift", + "systemverilog": "SystemVerilog", + "tcl": "Tcl", + "tcsh": "Tcsh", + "tea": "Tea", + "terra": "Terra", + "tex": "TeX", + "text": "Text", + "textile": "Textile", + "thrift": "Thrift", + "ti_program": "TI Program", + "tl": "Type Language", + "tla": "TLA", + "toml": "TOML", + "ts": "TypeScript", + "turing": "Turing", + "turtle": "Turtle", + "twig": "Twig", + "txl": "TXL", + "type_language": "Type Language", + "typescript": "TypeScript", + "udiff": "Diff", + "unified_parallel_c": "Unified Parallel C", + "unity3d_asset": "Unity3D Asset", + "unix_assembly": "Unix Assembly", + "uno": "Uno", + "unrealscript": "UnrealScript", + "ur": "UrWeb", + "ur/web": "UrWeb", + "urweb": "UrWeb", + "vala": "Vala", + "vb.net": "Visual Basic", + "vbnet": "Visual Basic", + "vcl": "VCL", + "verilog": "Verilog", + "vhdl": "VHDL", + "vim": "Vim script", + "vim_script": "Vim script", + "viml": "Vim script", + "visual_basic": "Visual Basic", + "volt": "Volt", + "vue": "Vue", + "wasm": "WebAssembly", + "wast": "WebAssembly", + "wavefront_material": "Wavefront Material", + "wavefront_object": "Wavefront Object", + "web_ontology_language": "Web Ontology Language", + "webassembly": "WebAssembly", + "webidl": "WebIDL", + "winbatch": "Batchfile", + "wisp": "wisp", + "world_of_warcraft_addon_data": "World of Warcraft Addon Data", + "wsdl": "XML", + "x10": "X10", + "xbase": "xBase", + "xc": "XC", + "xcompose": "XCompose", + "xhtml": "HTML", + "xml": "XML", + "xml+genshi": "Genshi", + "xml+kid": "Genshi", + "xojo": "Xojo", + "xpages": "XPages", + "xpm": "XPM", + "xproc": "XProc", + "xquery": "XQuery", + "xs": "XS", + "xsd": "XML", + "xsl": "XSLT", + "xslt": "XSLT", + "xten": "X10", + "xtend": "Xtend", + "yacc": "Yacc", + "yaml": "YAML", + "yang": "YANG", + "yml": "YAML", + "zephir": "Zephir", + "zimpl": "Zimpl", + "zsh": "Shell", +} diff --git a/vendor/gopkg.in/src-d/enry.v1/data/commit.go b/vendor/gopkg.in/src-d/enry.v1/data/commit.go new file mode 100644 index 000000000..79fd398af --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/data/commit.go @@ -0,0 +1,8 @@ +package data + +// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/enry.v1/internal/code-generator +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: 4cd558c37482e8d2c535d8107f2d11b49afbc5b5 + +// linguist's commit from which files were generated. +var LinguistCommit = "4cd558c37482e8d2c535d8107f2d11b49afbc5b5" diff --git a/vendor/gopkg.in/src-d/enry.v1/data/content.go b/vendor/gopkg.in/src-d/enry.v1/data/content.go new file mode 100644 index 000000000..42c7ab3e5 --- /dev/null +++ b/vendor/gopkg.in/src-d/enry.v1/data/content.go @@ -0,0 +1,585 @@ +package data + +// CODE GENERATED AUTOMATICALLY WITH gopkg.in/src-d/enry.v1/internal/code-generator +// THIS FILE SHOULD NOT BE EDITED BY HAND +// Extracted from github/linguist commit: 4cd558c37482e8d2c535d8107f2d11b49afbc5b5 + +import "gopkg.in/toqueteos/substring.v1" + +type languageMatcher func([]byte) []string + +var ContentMatchers = map[string]languageMatcher{ + ".asc": func(i []byte) []string { + if asc_PublicKey_Matcher_0.Match(string(i)) { + return []string{"Public Key"} + } else if asc_AsciiDoc_Matcher_0.Match(string(i)) { + return []string{"AsciiDoc"} + } else if asc_AGSScript_Matcher_0.Match(string(i)) { + return []string{"AGS Script"} + } + + return nil + }, + ".bb": func(i []byte) []string { + if bb_BlitzBasic_Matcher_0.Match(string(i)) || bb_BlitzBasic_Matcher_1.Match(string(i)) { + return []string{"BlitzBasic"} + } else if bb_BitBake_Matcher_0.Match(string(i)) { + return []string{"BitBake"} + } + + return nil + }, + ".builds": func(i []byte) []string { + if builds_XML_Matcher_0.Match(string(i)) { + return []string{"XML"} + } + + return []string{"Text"} + }, + ".ch": func(i []byte) []string { + if ch_xBase_Matcher_0.Match(string(i)) { + return []string{"xBase"} + } + + return nil + }, + ".cl": func(i []byte) []string { + if cl_CommonLisp_Matcher_0.Match(string(i)) { + return []string{"Common Lisp"} + } else if cl_Cool_Matcher_0.Match(string(i)) { + return []string{"Cool"} + } else if cl_OpenCL_Matcher_0.Match(string(i)) { + return []string{"OpenCL"} + } + + return nil + }, + ".cls": func(i []byte) []string { + if cls_TeX_Matcher_0.Match(string(i)) { + return []string{"TeX"} + } + + return nil + }, + ".cs": func(i []byte) []string { + if cs_Smalltalk_Matcher_0.Match(string(i)) { + return []string{"Smalltalk"} + } else if cs_CSharp_Matcher_0.Match(string(i)) || cs_CSharp_Matcher_1.Match(string(i)) { + return []string{"C#"} + } + + return nil + }, + ".d": func(i []byte) []string { + if d_D_Matcher_0.Match(string(i)) { + return []string{"D"} + } else if d_DTrace_Matcher_0.Match(string(i)) { + return []string{"DTrace"} + } else if d_Makefile_Matcher_0.Match(string(i)) { + return []string{"Makefile"} + } + + return nil + }, + ".ecl": func(i []byte) []string { + if ecl_ECLiPSe_Matcher_0.Match(string(i)) { + return []string{"ECLiPSe"} + } else if ecl_ECL_Matcher_0.Match(string(i)) { + return []string{"ECL"} + } + + return nil + }, + ".es": func(i []byte) []string { + if es_Erlang_Matcher_0.Match(string(i)) { + return []string{"Erlang"} + } + + return nil + }, + ".f": func(i []byte) []string { + if f_Forth_Matcher_0.Match(string(i)) { + return []string{"Forth"} + } else if f_FilebenchWML_Matcher_0.Match(string(i)) { + return []string{"Filebench WML"} + } else if f_Fortran_Matcher_0.Match(string(i)) { + return []string{"Fortran"} + } + + return nil + }, + ".for": func(i []byte) []string { + if for_Forth_Matcher_0.Match(string(i)) { + return []string{"Forth"} + } else if for_Fortran_Matcher_0.Match(string(i)) { + return []string{"Fortran"} + } + + return nil + }, + ".fr": func(i []byte) []string { + if fr_Forth_Matcher_0.Match(string(i)) { + return []string{"Forth"} + } else if fr_Frege_Matcher_0.Match(string(i)) { + return []string{"Frege"} + } + + return []string{"Text"} + }, + ".fs": func(i []byte) []string { + if fs_Forth_Matcher_0.Match(string(i)) { + return []string{"Forth"} + } else if fs_FSharp_Matcher_0.Match(string(i)) { + return []string{"F#"} + } else if fs_GLSL_Matcher_0.Match(string(i)) { + return []string{"GLSL"} + } else if fs_Filterscript_Matcher_0.Match(string(i)) { + return []string{"Filterscript"} + } + + return nil + }, + ".gs": func(i []byte) []string { + if gs_Gosu_Matcher_0.Match(string(i)) { + return []string{"Gosu"} + } + + return nil + }, + ".h": func(i []byte) []string { + if h_ObjectiveDashC_Matcher_0.Match(string(i)) { + return []string{"Objective-C"} + } else if h_CPlusPlus_Matcher_0.Match(string(i)) || h_CPlusPlus_Matcher_1.Match(string(i)) || h_CPlusPlus_Matcher_2.Match(string(i)) || h_CPlusPlus_Matcher_3.Match(string(i)) || h_CPlusPlus_Matcher_4.Match(string(i)) || h_CPlusPlus_Matcher_5.Match(string(i)) || h_CPlusPlus_Matcher_6.Match(string(i)) { + return []string{"C++"} + } + + return nil + }, + ".inc": func(i []byte) []string { + if inc_PHP_Matcher_0.Match(string(i)) { + return []string{"PHP"} + } else if inc_POVDashRaySDL_Matcher_0.Match(string(i)) { + return []string{"POV-Ray SDL"} + } + + return nil + }, + ".l": func(i []byte) []string { + if l_CommonLisp_Matcher_0.Match(string(i)) { + return []string{"Common Lisp"} + } else if l_Lex_Matcher_0.Match(string(i)) { + return []string{"Lex"} + } else if l_Roff_Matcher_0.Match(string(i)) { + return []string{"Roff"} + } else if l_PicoLisp_Matcher_0.Match(string(i)) { + return []string{"PicoLisp"} + } + + return nil + }, + ".ls": func(i []byte) []string { + if ls_LoomScript_Matcher_0.Match(string(i)) { + return []string{"LoomScript"} + } + + return []string{"LiveScript"} + }, + ".lsp": func(i []byte) []string { + if lsp_CommonLisp_Matcher_0.Match(string(i)) { + return []string{"Common Lisp"} + } else if lsp_NewLisp_Matcher_0.Match(string(i)) { + return []string{"NewLisp"} + } + + return nil + }, + ".lisp": func(i []byte) []string { + if lisp_CommonLisp_Matcher_0.Match(string(i)) { + return []string{"Common Lisp"} + } else if lisp_NewLisp_Matcher_0.Match(string(i)) { + return []string{"NewLisp"} + } + + return nil + }, + ".m": func(i []byte) []string { + if m_ObjectiveDashC_Matcher_0.Match(string(i)) { + return []string{"Objective-C"} + } else if m_Mercury_Matcher_0.Match(string(i)) { + return []string{"Mercury"} + } else if m_MUF_Matcher_0.Match(string(i)) { + return []string{"MUF"} + } else if m_M_Matcher_0.Match(string(i)) { + return []string{"M"} + } else if m_Mathematica_Matcher_0.Match(string(i)) { + return []string{"Mathematica"} + } else if m_Matlab_Matcher_0.Match(string(i)) { + return []string{"Matlab"} + } else if m_Limbo_Matcher_0.Match(string(i)) { + return []string{"Limbo"} + } + + return nil + }, + ".md": func(i []byte) []string { + if md_Markdown_Matcher_0.Match(string(i)) || md_Markdown_Matcher_1.Match(string(i)) { + return []string{"Markdown"} + } else if md_GCCMachineDescription_Matcher_0.Match(string(i)) { + return []string{"GCC Machine Description"} + } + + return []string{"Markdown"} + }, + ".ml": func(i []byte) []string { + if ml_OCaml_Matcher_0.Match(string(i)) { + return []string{"OCaml"} + } else if ml_StandardML_Matcher_0.Match(string(i)) { + return []string{"Standard ML"} + } + + return nil + }, + ".mod": func(i []byte) []string { + if mod_XML_Matcher_0.Match(string(i)) { + return []string{"XML"} + } else if mod_ModulaDash2_Matcher_0.Match(string(i)) || mod_ModulaDash2_Matcher_1.Match(string(i)) { + return []string{"Modula-2"} + } + + return []string{"Linux Kernel Module", "AMPL"} + }, + ".ms": func(i []byte) []string { + if ms_Roff_Matcher_0.Match(string(i)) { + return []string{"Roff"} + } + + return []string{"MAXScript"} + }, + ".n": func(i []byte) []string { + if n_Roff_Matcher_0.Match(string(i)) { + return []string{"Roff"} + } else if n_Nemerle_Matcher_0.Match(string(i)) { + return []string{"Nemerle"} + } + + return nil + }, + ".ncl": func(i []byte) []string { + if ncl_Text_Matcher_0.Match(string(i)) { + return []string{"Text"} + } + + return nil + }, + ".nl": func(i []byte) []string { + if nl_NL_Matcher_0.Match(string(i)) { + return []string{"NL"} + } + + return []string{"NewLisp"} + }, + ".php": func(i []byte) []string { + if php_Hack_Matcher_0.Match(string(i)) { + return []string{"Hack"} + } else if php_PHP_Matcher_0.Match(string(i)) { + return []string{"PHP"} + } + + return nil + }, + ".pl": func(i []byte) []string { + if pl_Prolog_Matcher_0.Match(string(i)) { + return []string{"Prolog"} + } else if pl_Perl_Matcher_0.Match(string(i)) { + return []string{"Perl"} + } else if pl_Perl6_Matcher_0.Match(string(i)) { + return []string{"Perl 6"} + } + + return nil + }, + ".pm": func(i []byte) []string { + if pm_Perl6_Matcher_0.Match(string(i)) { + return []string{"Perl 6"} + } else if pm_Perl_Matcher_0.Match(string(i)) { + return []string{"Perl"} + } else if pm_XPM_Matcher_0.Match(string(i)) { + return []string{"XPM"} + } + + return nil + }, + ".pod": func(i []byte) []string { + if pod_Pod_Matcher_0.Match(string(i)) { + return []string{"Pod"} + } + + return []string{"Perl"} + }, + "Pod": func(i []byte) []string { + if Pod_Pod_Matcher_0.Match(string(i)) { + return []string{"Pod"} + } + + return []string{"Perl"} + }, + "Perl": func(i []byte) []string { + if Perl_Pod_Matcher_0.Match(string(i)) { + return []string{"Pod"} + } + + return []string{"Perl"} + }, + ".pro": func(i []byte) []string { + if pro_Prolog_Matcher_0.Match(string(i)) { + return []string{"Prolog"} + } else if pro_INI_Matcher_0.Match(string(i)) { + return []string{"INI"} + } else if pro_QMake_Matcher_0.Match(string(i)) && pro_QMake_Matcher_1.Match(string(i)) { + return []string{"QMake"} + } else if pro_IDL_Matcher_0.Match(string(i)) { + return []string{"IDL"} + } + + return nil + }, + ".props": func(i []byte) []string { + if props_XML_Matcher_0.Match(string(i)) { + return []string{"XML"} + } else if props_INI_Matcher_0.Match(string(i)) { + return []string{"INI"} + } + + return nil + }, + ".r": func(i []byte) []string { + if r_Rebol_Matcher_0.Match(string(i)) { + return []string{"Rebol"} + } else if r_R_Matcher_0.Match(string(i)) { + return []string{"R"} + } + + return nil + }, + ".rno": func(i []byte) []string { + if rno_RUNOFF_Matcher_0.Match(string(i)) { + return []string{"RUNOFF"} + } else if rno_Roff_Matcher_0.Match(string(i)) { + return []string{"Roff"} + } + + return nil + }, + ".rpy": func(i []byte) []string { + if rpy_Python_Matcher_0.Match(string(i)) { + return []string{"Python"} + } + + return []string{"Ren'Py"} + }, + ".rs": func(i []byte) []string { + if rs_Rust_Matcher_0.Match(string(i)) { + return []string{"Rust"} + } else if rs_RenderScript_Matcher_0.Match(string(i)) { + return []string{"RenderScript"} + } + + return nil + }, + ".sc": func(i []byte) []string { + if sc_SuperCollider_Matcher_0.Match(string(i)) || sc_SuperCollider_Matcher_1.Match(string(i)) || sc_SuperCollider_Matcher_2.Match(string(i)) { + return []string{"SuperCollider"} + } else if sc_Scala_Matcher_0.Match(string(i)) || sc_Scala_Matcher_1.Match(string(i)) || sc_Scala_Matcher_2.Match(string(i)) { + return []string{"Scala"} + } + + return nil + }, + ".sql": func(i []byte) []string { + if sql_PLpgSQL_Matcher_0.Match(string(i)) || sql_PLpgSQL_Matcher_1.Match(string(i)) || sql_PLpgSQL_Matcher_2.Match(string(i)) { + return []string{"PLpgSQL"} + } else if sql_SQLPL_Matcher_0.Match(string(i)) || sql_SQLPL_Matcher_1.Match(string(i)) { + return []string{"SQLPL"} + } else if sql_PLSQL_Matcher_0.Match(string(i)) || sql_PLSQL_Matcher_1.Match(string(i)) { + return []string{"PLSQL"} + } else if sql_SQL_Matcher_0.Match(string(i)) { + return []string{"SQL"} + } + + return nil + }, + ".srt": func(i []byte) []string { + if srt_SubRipText_Matcher_0.Match(string(i)) { + return []string{"SubRip Text"} + } + + return nil + }, + ".t": func(i []byte) []string { + if t_Turing_Matcher_0.Match(string(i)) { + return []string{"Turing"} + } else if t_Perl6_Matcher_0.Match(string(i)) { + return []string{"Perl 6"} + } else if t_Perl_Matcher_0.Match(string(i)) { + return []string{"Perl"} + } + + return nil + }, + ".toc": func(i []byte) []string { + if toc_WorldofWarcraftAddonData_Matcher_0.Match(string(i)) { + return []string{"World of Warcraft Addon Data"} + } else if toc_TeX_Matcher_0.Match(string(i)) { + return []string{"TeX"} + } + + return nil + }, + ".ts": func(i []byte) []string { + if ts_XML_Matcher_0.Match(string(i)) { + return []string{"XML"} + } + + return []string{"TypeScript"} + }, + ".tst": func(i []byte) []string { + if tst_GAP_Matcher_0.Match(string(i)) { + return []string{"GAP"} + } + + return []string{"Scilab"} + }, + ".tsx": func(i []byte) []string { + if tsx_TypeScript_Matcher_0.Match(string(i)) { + return []string{"TypeScript"} + } else if tsx_XML_Matcher_0.Match(string(i)) { + return []string{"XML"} + } + + return nil + }, +} + +var ( + asc_PublicKey_Matcher_0 = substring.Regexp(`(?m)^(----[- ]BEGIN|ssh-(rsa|dss)) `) + asc_AsciiDoc_Matcher_0 = substring.Regexp(`(?m)^[=-]+(\s|\n)|{{[A-Za-z]`) + asc_AGSScript_Matcher_0 = substring.Regexp(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`) + bb_BlitzBasic_Matcher_0 = substring.Regexp(`(?m)^\s*; `) + bb_BlitzBasic_Matcher_1 = substring.Regexp(`(?m)End Function`) + bb_BitBake_Matcher_0 = substring.Regexp(`(?m)^\s*(# |include|require)\b`) + builds_XML_Matcher_0 = substring.Regexp(`(?mi)^(\s*)()`) + f_Forth_Matcher_0 = substring.Regexp(`(?m)^: `) + f_FilebenchWML_Matcher_0 = substring.Regexp(`(?m)flowop`) + f_Fortran_Matcher_0 = substring.Regexp(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) + for_Forth_Matcher_0 = substring.Regexp(`(?m)^: `) + for_Fortran_Matcher_0 = substring.Regexp(`(?mi)^([c*][^abd-z]| (subroutine|program|end|data)\s|\s*!)`) + fr_Forth_Matcher_0 = substring.Regexp(`(?m)^(: |also |new-device|previous )`) + fr_Frege_Matcher_0 = substring.Regexp(`(?m)^\s*(import|module|package|data|type) `) + fs_Forth_Matcher_0 = substring.Regexp(`(?m)^(: |new-device)`) + fs_FSharp_Matcher_0 = substring.Regexp(`(?m)^\s*(#light|import|let|module|namespace|open|type)`) + fs_GLSL_Matcher_0 = substring.Regexp(`(?m)^\s*(#version|precision|uniform|varying|vec[234])`) + fs_Filterscript_Matcher_0 = substring.Regexp(`(?m)#include|#pragma\s+(rs|version)|__attribute__`) + gs_Gosu_Matcher_0 = substring.Regexp(`(?m)^uses java\.`) + h_ObjectiveDashC_Matcher_0 = substring.Regexp(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`) + h_CPlusPlus_Matcher_0 = substring.Regexp(`(?m)^\s*#\s*include <(cstdint|string|vector|map|list|array|bitset|queue|stack|forward_list|unordered_map|unordered_set|(i|o|io)stream)>`) + h_CPlusPlus_Matcher_1 = substring.Regexp(`(?m)^\s*template\s*<`) + h_CPlusPlus_Matcher_2 = substring.Regexp(`(?m)^[ \t]*try`) + h_CPlusPlus_Matcher_3 = substring.Regexp(`(?m)^[ \t]*catch\s*\(`) + h_CPlusPlus_Matcher_4 = substring.Regexp(`(?m)^[ \t]*(class|(using[ \t]+)?namespace)\s+\w+`) + h_CPlusPlus_Matcher_5 = substring.Regexp(`(?m)^[ \t]*(private|public|protected):$`) + h_CPlusPlus_Matcher_6 = substring.Regexp(`(?m)std::\w+`) + inc_PHP_Matcher_0 = substring.Regexp(`(?m)^<\?(?:php)?`) + inc_POVDashRaySDL_Matcher_0 = substring.Regexp(`(?m)^\s*#(declare|local|macro|while)\s`) + l_CommonLisp_Matcher_0 = substring.Regexp(`(?m)\(def(un|macro)\s`) + l_Lex_Matcher_0 = substring.Regexp(`(?m)^(%[%{}]xs|<.*>)`) + l_Roff_Matcher_0 = substring.Regexp(`(?mi)^\.[a-z][a-z](\s|$)`) + l_PicoLisp_Matcher_0 = substring.Regexp(`(?m)^\((de|class|rel|code|data|must)\s`) + ls_LoomScript_Matcher_0 = substring.Regexp(`(?m)^\s*package\s*[\w\.\/\*\s]*\s*{`) + lsp_CommonLisp_Matcher_0 = substring.Regexp(`(?mi)^\s*\((defun|in-package|defpackage) `) + lsp_NewLisp_Matcher_0 = substring.Regexp(`(?m)^\s*\(define `) + lisp_CommonLisp_Matcher_0 = substring.Regexp(`(?mi)^\s*\((defun|in-package|defpackage) `) + lisp_NewLisp_Matcher_0 = substring.Regexp(`(?m)^\s*\(define `) + m_ObjectiveDashC_Matcher_0 = substring.Regexp(`(?m)^\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\b|#import\s+.+\.h[">])`) + m_Mercury_Matcher_0 = substring.Regexp(`(?m):- module`) + m_MUF_Matcher_0 = substring.Regexp(`(?m)^: `) + m_M_Matcher_0 = substring.Regexp(`(?m)^\s*;`) + m_Mathematica_Matcher_0 = substring.Regexp(`(?m)\*\)$`) + m_Matlab_Matcher_0 = substring.Regexp(`(?m)^\s*%`) + m_Limbo_Matcher_0 = substring.Regexp(`(?m)^\w+\s*:\s*module\s*{`) + md_Markdown_Matcher_0 = substring.Regexp(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`) + md_Markdown_Matcher_1 = substring.Regexp(`^$`) + md_GCCMachineDescription_Matcher_0 = substring.Regexp(`(?m)^(;;|\(define_)`) + ml_OCaml_Matcher_0 = substring.Regexp(`(?m)(^\s*module)|let rec |match\s+(\S+\s)+with`) + ml_StandardML_Matcher_0 = substring.Regexp(`(?m)=> |case\s+(\S+\s)+of`) + mod_XML_Matcher_0 = substring.Regexp(`(?m))\s*(\d{2}:\d{2}:\d{2},\d{3})$`) + t_Turing_Matcher_0 = substring.Regexp(`(?m)^\s*%[ \t]+|^\s*var\s+\w+\s*:=\s*\w+`) + t_Perl6_Matcher_0 = substring.Regexp(`(?m)^\s*(?:use\s+v6\s*;|\bmodule\b|\b(?:my\s+)?class\b)`) + t_Perl_Matcher_0 = substring.Regexp(`(?m)\buse\s+(?:strict\b|v?5\.)`) + toc_WorldofWarcraftAddonData_Matcher_0 = substring.Regexp(`(?m)^## |@no-lib-strip@`) + toc_TeX_Matcher_0 = substring.Regexp(`(?m)^\\(contentsline|defcounter|beamer|boolfalse)`) + ts_XML_Matcher_0 = substring.Regexp(`(?m) `) + tsx_TypeScript_Matcher_0 = substring.Regexp(`(?m)^\s*(import.+(from\s+|require\()['"]react|\/\/\/\s*": -6.727180, + "?": -8.978471, + "DOM": -8.978471, + "DOM.": -9.671619, + "HTML": -8.978471, + "HTML.": -8.978471, + "Null": -9.671619, + "XML": -7.273723, + "XML.": -7.725708, + "_": -7.474394, + "А": -8.285324, + "Б": -8.062181, + "В": -4.701805, + "Г": -8.285324, + "Д": -4.722859, + "Е": -5.107270, + "Ж": -9.671619, + "З": -4.584022, + "И": -5.289592, + "К": -4.310326, + "Л": -6.237631, + "М": -6.270421, + "Н": -4.590214, + "О": -4.641181, + "П": -4.001738, + "Р": -5.842977, + "С": -4.435177, + "Т": -4.403760, + "У": -6.060701, + "Ф": -5.982739, + "Х": -8.978471, + "Ц": -5.611176, + "Ч": -6.580576, + "Ш": -6.627096, + "Э": -5.821471, + "а": -2.506125, + "б": -4.291721, + "в": -3.860478, + "г": -4.804084, + "д": -4.036829, + "е": -2.541520, + "ж": -5.139019, + "з": -4.827431, + "и": -2.859273, + "й": -4.688012, + "к": -3.479256, + "л": -3.214849, + "м": -4.073197, + "н": -2.957448, + "о": -2.573243, + "п": -4.273456, + "р": -3.199272, + "с": -3.569060, + "т": -2.863684, + "у": -4.029711, + "ф": -5.821471, + "х": -6.237631, + "ц": -4.319760, + "ч": -4.654339, + "ш": -7.369033, + "щ": -5.528484, + "ъ": -5.982739, + "ы": -3.805150, + "ь": -4.029711, + "ю": -6.116270, + "я": -4.565673, + "\ufeff": -8.978471, + }, + "ABAP": map[string]float64{ + "!": -4.637712, + "\"": -7.202661, + "(": -5.816367, + ")": -5.816367, + "*": -3.619142, + "+": -5.005437, + "-": -0.322277, + ".": -5.005437, + "<-()]>": -6.509514, + "": -6.509514, + "": -5.816367, + "": -6.104049, + "=": -4.900076, + ">": -5.123220, + "ABAP_BOOL": -7.202661, + "C": -7.202661, + "CLASS": -6.509514, + "CL_CSV_PARSER": -5.410902, + "CONSTRUCTOR": -7.202661, + "CSV": -7.202661, + "CSVSTRING": -7.202661, + "CX": -7.202661, + "CX_CSV_PARSE_ERROR": -7.202661, + "DEFINITION": -6.509514, + "DELEGATE": -7.202661, + "Get": -7.202661, + "IF_CSV_PARSER_DELEGATE": -7.202661, + "IMPLEMENTATION": -6.509514, + "Instance": -6.509514, + "Method": -6.509514, + "Parse": -7.202661, + "Private": -7.202661, + "Public": -7.202661, + "REF": -7.202661, + "RETURNING": -7.202661, + "SEPARATOR": -7.202661, + "SIGNATURE": -6.509514, + "SKIP_FIRST_LINE": -7.202661, + "STRING": -7.202661, + "STRINGTAB": -6.104049, + "Space": -6.509514, + "TO": -7.202661, + "TYPE": -5.256751, + "This": -7.202661, + "[": -5.593223, + "]": -5.816367, + "_LINES": -7.202661, + "_csvstring": -6.509514, + "_delegate": -7.202661, + "_lines": -7.202661, + "_lines.": -7.202661, + "_parse_line": -6.509514, + "_separator": -7.202661, + "_skip_first_line": -7.202661, + "_textindicator": -7.202661, + "`": -5.816367, + "abap": -7.202661, + "abap_bool": -6.509514, + "abap_true.": -6.509514, + "an": -7.202661, + "append": -6.509514, + "assigning": -7.202661, + "at": -6.509514, + "c": -6.509514, + "char": -6.509514, + "cl_abap_char_utilities": -7.202661, + "cl_csv_parser": -6.509514, + "cl_object": -7.202661, + "class": -6.509514, + "clear": -7.202661, + "concatenate": -5.816367, + "constants": -7.202661, + "constructor": -6.509514, + "constructor.": -7.202661, + "cr_lf": -7.202661, + "create": -7.202661, + "csv": -7.202661, + "csvstring": -7.202661, + "csvstring.": -7.202661, + "csvvalue": -5.410902, + "csvvalue.": -5.593223, + "csvvalues.": -6.509514, + "cx_csv_parse_error": -6.509514, + "data": -6.104049, + "definition": -7.202661, + "delegate": -7.202661, + "delegate.": -7.202661, + "do": -6.104049, + "e": -7.202661, + "else.": -5.816367, + "endclass.": -7.202661, + "endif.": -5.410902, + "endmethod.": -6.509514, + "endwhile.": -6.509514, + "error": -7.202661, + "exception": -7.202661, + "exporting": -7.202661, + "field": -7.202661, + "files": -6.104049, + "final": -7.202661, + "formatting": -7.202661, + "from": -7.202661, + "here": -6.104049, + "if_csv_parser_delegate": -7.202661, + "implementation.": -7.202661, + "importing": -7.202661, + "in": -7.202661, + "include": -6.104049, + "indicates": -7.202661, + "inheriting": -7.202661, + "into": -5.410902, + "is_first_line": -7.202661, + "line": -7.202661, + "lines": -5.816367, + "loop": -7.202661, + "message": -6.509514, + "method": -6.509514, + "methods": -6.509514, + "msg.": -6.509514, + "not": -6.104049, + "of": -6.509514, + "other": -6.104049, + "parse": -6.509514, + "pools": -7.202661, + "pos": -6.509514, + "private": -7.202661, + "protected": -7.202661, + "public": -6.104049, + "raise": -7.202661, + "raising": -7.202661, + "ref": -7.202661, + "returning.": -7.202661, + "section.": -6.104049, + "separator": -7.202661, + "separator.": -7.202661, + "skip_first_line": -7.202661, + "skip_first_line.": -7.202661, + "source": -6.104049, + "split": -7.202661, + "standard": -6.509514, + "string": -7.202661, + "string.": -6.104049, + "super": -7.202661, + "symbols": -7.202661, + "table": -6.104049, + "text_ended": -7.202661, + "the": -6.104049, + "to": -6.104049, + "type": -4.804766, + "value": -6.509514, + "values": -6.509514, + "|": -5.256751, + }, + "ABNF": map[string]float64{ + "%": -3.171745, + "'": -7.142037, + "(": -4.251665, + ")": -4.197598, + "*": -4.944812, + "*DIGIT": -7.142037, + "*basic": -6.448889, + "*literal": -7.142037, + "*newline": -7.142037, + "*non": -7.142037, + "+": -4.744141, + ",": -5.062595, + "-": -1.785450, + ".": -6.043424, + "/": -2.983153, + "//github.com/toml": -7.142037, + "//www.ietf.org/rfc/rfc": -7.142037, + ":": -5.755742, + ";": -2.588160, + "=": -2.747587, + "A": -5.196126, + "ABNF": -6.448889, + "ALPHA": -6.448889, + "Apostrophe": -7.142037, + "Array": -6.448889, + "B": -5.196126, + "B.": -7.142037, + "Basic": -6.043424, + "Boolean": -7.142037, + "Built": -7.142037, + "C": -5.350277, + "C.": -7.142037, + "CRLF": -7.142037, + "Comma": -6.448889, + "Comment": -7.142037, + "D": -4.839451, + "D.": -6.448889, + "DIGIT": -4.433986, + "Datetime": -7.142037, + "Double": -6.448889, + "E": -5.755742, + "Float": -7.142037, + "HEXDIG": -6.043424, + "Horizontal": -7.142037, + "Inline": -7.142037, + "Integer": -7.142037, + "Key": -7.142037, + "LF": -7.142037, + "Left": -7.142037, + "License": -7.142037, + "Literal": -6.448889, + "MIT": -7.142037, + "Multiline": -6.448889, + "Newline": -7.142037, + "Period": -7.142037, + "RFC": -6.448889, + "Right": -7.142037, + "See": -7.142037, + "Source": -7.142037, + "Space": -7.142037, + "Standard": -7.142037, + "String": -5.532599, + "Strings": -7.142037, + "TOML": -6.448889, + "Table": -5.755742, + "This": -7.142037, + "U": -4.839451, + "UXXXXXXXX": -7.142037, + "Value": -7.142037, + "Whitespace": -7.142037, + "XXXX": -7.142037, + "XXXXXXXX": -7.142037, + "Z": -6.448889, + "[": -4.577087, + "\\": -6.448889, + "]": -4.577087, + "_": -6.448889, + "a": -6.448889, + "according": -7.142037, + "an": -7.142037, + "apostraphe": -5.350277, + "array": -4.251665, + "as": -7.142037, + "attempt": -7.142037, + "b": -7.142037, + "backspace": -7.142037, + "based": -6.448889, + "basic": -4.502979, + "body": -5.755742, + "boolean": -6.448889, + "bracket": -5.755742, + "carriage": -7.142037, + "char": -5.062595, + "clarity": -7.142037, + "close": -5.062595, + "comment": -5.062595, + "date": -4.839451, + "decimal": -6.448889, + "define": -7.142037, + "defined": -6.448889, + "delim": -5.350277, + "digit": -6.448889, + "e": -6.043424, + "empty": -6.043424, + "eol": -7.142037, + "escape": -6.448889, + "escaped": -7.142037, + "exp": -6.043424, + "expression": -6.043424, + "f": -7.142037, + "false": -6.043424, + "feed": -6.448889, + "float": -6.448889, + "for": -7.142037, + "form": -7.142037, + "frac": -6.043424, + "full": -5.755742, + "fullyear": -6.448889, + "grammar": -7.142037, + "here": -7.142037, + "hour": -6.043424, + "http": -7.142037, + "https": -7.142037, + "in": -5.755742, + "inline": -4.577087, + "int": -5.755742, + "integer": -5.755742, + "is": -7.142037, + "key": -4.433986, + "keyval": -5.350277, + "keyvals": -5.532599, + "lang/toml": -7.142037, + "leap": -7.142037, + "left": -7.142037, + "line": -7.142037, + "literal": -4.657130, + "mark": -4.944812, + "mday": -6.448889, + "minus": -6.448889, + "minute": -6.043424, + "ml": -4.146304, + "month": -6.448889, + "month/year": -7.142037, + "n": -7.142037, + "newline": -5.532599, + "newlines": -5.532599, + "non": -6.043424, + "numoffset": -6.448889, + "offset": -6.448889, + "on": -6.448889, + "open": -5.062595, + "pairs": -7.142037, + "partial": -6.448889, + "plus": -6.448889, + "point": -6.448889, + "prefixable": -6.448889, + "quotation": -4.944812, + "quoted": -6.448889, + "r": -7.142037, + "reproduced": -7.142037, + "return": -7.142037, + "reverse": -7.142037, + "right": -7.142037, + "rules": -7.142037, + "secfrac": -6.448889, + "second": -6.043424, + "sep": -4.657130, + "solidus": -6.448889, + "square": -5.755742, + "start": -6.448889, + "std": -5.350277, + "string": -4.369448, + "symbol": -6.448889, + "t": -7.142037, + "tab": -6.448889, + "table": -3.740839, + "terms": -7.142037, + "the": -7.142037, + "time": -4.146304, + "to": -6.448889, + "toml": -7.142037, + "true": -6.043424, + "txt": -7.142037, + "uXXXX": -7.142037, + "underscore": -6.043424, + "unescaped": -6.043424, + "unquoted": -6.448889, + "val": -5.350277, + "values": -6.043424, + "ws": -4.006542, + "x": -3.171745, + "z": -6.448889, + "zero": -6.448889, + "{": -7.142037, + "}": -7.142037, + }, + "AGS Script": map[string]float64{ + "!": -5.850405, + "#define": -7.236699, + "&&": -6.138087, + "(": -2.606836, + ")": -2.606836, + "*control": -4.710971, + "*theGui": -7.929846, + "+": -5.983936, + ",": -3.978603, + "-": -2.601970, + "//": -5.531951, + "//****************************************************************************************************": -5.850405, + "//But": -7.929846, + "//Ctrl": -7.929846, + "//START": -7.929846, + "//Use": -7.929846, + "//if": -7.929846, + ":": -7.236699, + ";": -2.776555, + "<": -7.929846, + "=": -1.114206, + ">": -7.236699, + "CONTROL": -7.929846, + "DISTANCE": -4.710971, + "Debug": -6.543552, + "DeleteSaveSlot": -7.929846, + "Display": -6.543552, + "ESC": -7.929846, + "EventType": -7.929846, + "FUNCTIONS": -7.929846, + "GUI": -7.929846, + "GUIControl": -4.710971, + "IsGamePaused": -6.831234, + "IsInterfaceEnabled": -7.236699, + "IsKeyPressed": -5.096633, + "IsSpeechVoxAvailable": -7.929846, + "KeyboardMovement": -7.236699, + "KeyboardMovement_CurrentDirection": -5.983936, + "KeyboardMovement_Directions": -6.543552, + "KeyboardMovement_KeyDown": -6.320409, + "KeyboardMovement_KeyDownLeft": -6.831234, + "KeyboardMovement_KeyDownRight": -6.831234, + "KeyboardMovement_KeyLeft": -6.320409, + "KeyboardMovement_KeyRight": -6.320409, + "KeyboardMovement_KeyStop": -6.831234, + "KeyboardMovement_KeyUp": -6.320409, + "KeyboardMovement_KeyUpLeft": -6.831234, + "KeyboardMovement_KeyUpRight": -6.831234, + "KeyboardMovement_Mode": -6.543552, + "KeyboardMovement_Modes": -6.543552, + "KeyboardMovement_VERSION": -7.929846, + "MouseButton": -4.838804, + "OF": -7.929846, + "PANEL": -7.929846, + "ProcessClick": -7.929846, + "QuitGame": -7.236699, + "RestoreGameSlot": -7.929846, + "SaveGameSlot": -7.929846, + "SetMode": -7.236699, + "SetVoiceMode": -6.543552, + "System.SupportsGammaControl": -7.929846, + "System.Volume": -7.929846, + "W": -7.929846, + "Wait": -6.831234, + "[": -6.138087, + "]": -6.138087, + "appreciate.": -7.929846, + "area": -7.929846, + "as": -7.929846, + "btnAbout_OnClick": -7.929846, + "btnCancelRestore_OnClick": -7.929846, + "btnCancelSave_OnClick": -7.929846, + "btnDeleteSave_OnClick": -7.929846, + "btnIconAbout_Click": -7.929846, + "btnIconCurInv_Click": -7.929846, + "btnIconExit_Click": -7.929846, + "btnIconInv_Click": -7.929846, + "btnIconLoad": -7.929846, + "btnIconLoad_Click": -7.236699, + "btnIconSave": -7.929846, + "btnIconSave_Click": -7.236699, + "btnInvDown_Click": -7.929846, + "btnInvOK_Click": -7.929846, + "btnInvSelect_Click": -7.929846, + "btnInvUp_Click": -7.929846, + "btnLoad_OnClick": -7.929846, + "btnQuit_OnClick": -7.929846, + "btnRestoreGame_OnClick": -7.929846, + "btnResume_OnClick": -7.929846, + "btnSaveGame_OnClick": -7.236699, + "btnSave_OnClick": -7.929846, + "btnVoice.Text": -5.983936, + "btnVoice.Visible": -7.929846, + "button": -4.751793, + "cEgo_Interact": -7.929846, + "cEgo_Look": -7.929846, + "cEgo_Talk": -7.929846, + "cancel": -7.929846, + "close_restore_game_dialog": -6.831234, + "close_save_game_dialog": -6.831234, + "control": -7.929846, + "data": -7.929846, + "dx": -4.934114, + "dy": -4.934114, + "eEventLeaveRoom": -7.929846, + "eKeyCtrlA": -7.929846, + "eKeyCtrlS": -7.929846, + "eKeyCtrlV": -7.929846, + "eKeyCtrlW": -7.929846, + "eKeyCtrlX": -7.929846, + "eKeyEscape": -7.929846, + "eKeyboardMovement_Down": -6.320409, + "eKeyboardMovement_DownLeft": -6.320409, + "eKeyboardMovement_DownRight": -6.320409, + "eKeyboardMovement_Left": -6.320409, + "eKeyboardMovement_None": -7.236699, + "eKeyboardMovement_Pressing": -7.236699, + "eKeyboardMovement_Right": -6.320409, + "eKeyboardMovement_Stop": -5.732622, + "eKeyboardMovement_Tapping": -7.236699, + "eKeyboardMovement_Up": -6.320409, + "eKeyboardMovement_UpLeft": -6.320409, + "eKeyboardMovement_UpRight": -6.320409, + "eModeInteract": -7.929846, + "eModePickup": -7.929846, + "eModePointer": -6.320409, + "eModeTalkto": -7.929846, + "eModeUseinv": -7.236699, + "eModeWalkto": -7.929846, + "eMouseLeft": -6.831234, + "eMouseWheelNorth": -7.929846, + "eNoBlock": -7.236699, + "eSpeechTextOnly": -7.929846, + "eSpeechVoiceAndText": -7.236699, + "eSpeechVoiceOnly": -7.929846, + "else": -4.292260, + "enum": -7.236699, + "event": -7.236699, + "false": -4.985407, + "function": -4.240967, + "gControl_OnClick": -7.929846, + "gIconbar.Visible": -5.627261, + "gInventory.Visible": -7.929846, + "gPanel.Centre": -7.929846, + "gPanel.Visible": -5.983936, + "gRestartYN.Centre": -7.929846, + "gRestartYN.Visible": -7.236699, + "gRestoreGame.Visible": -7.236699, + "gSaveGame.Visible": -7.929846, + "game.debug_mode": -7.929846, + "gameSlotToSaveInto": -6.831234, + "give": -7.929846, + "i": -6.320409, + "if": -3.835502, + "import": -7.929846, + "initialize_control_panel": -7.929846, + "int": -5.096633, + "interface": -7.929846, + "interface_click": -7.929846, + "invCustomInv.ScrollDown": -7.929846, + "invCustomInv.ScrollUp": -7.929846, + "is": -7.929846, + "keyboard": -7.929846, + "keycode": -4.985407, + "lblVoice.Visible": -7.929846, + "lstRestoreGamesList.FillSaveGameList": -7.929846, + "lstRestoreGamesList.SaveGameSlots": -7.929846, + "lstRestoreGamesList.SelectedIndex": -7.236699, + "lstSaveGamesList.FillSaveGameList": -7.929846, + "lstSaveGamesList.ItemCount": -7.236699, + "lstSaveGamesList.Items": -6.831234, + "lstSaveGamesList.SaveGameSlots": -7.236699, + "lstSaveGamesList.SelectedIndex": -6.831234, + "lstSaveGamesList_OnSelectionCh": -7.929846, + "mode": -6.831234, + "mouse.Mode": -6.320409, + "mouse.UseDefaultGraphic": -5.983936, + "mouse.UseModeGraphic": -6.320409, + "mouse.x": -7.929846, + "mouse.y": -7.929846, + "move": -7.929846, + "much": -7.929846, + "newdirection": -4.168646, + "null": -7.929846, + "on_event": -7.929846, + "on_key_press": -7.929846, + "on_mouse_click": -7.929846, + "player.ActiveInventory": -7.929846, + "player.PlaceOnWalkableArea": -7.929846, + "player.StopMoving": -6.831234, + "player.WalkStraight": -7.236699, + "player.on": -7.236699, + "player.x": -7.236699, + "player.y": -7.236699, + "players": -7.929846, + "repeatedly_execute": -7.929846, + "restart.": -7.929846, + "return": -7.236699, + "shortcuts": -7.929846, + "show_inventory_window": -7.929846, + "show_restore_game_dialog": -7.236699, + "show_save_game_dialog": -7.929846, + "sldAudio.Value": -7.929846, + "sldAudio_OnChange": -7.929846, + "sldGamma_OnChange": -7.929846, + "sldVoice.Visible": -7.929846, + "sldVoice_OnChange": -7.929846, + "some": -7.929846, + "standard": -7.929846, + "static": -7.236699, + "struct": -7.929846, + "this": -7.929846, + "to": -7.236699, + "true": -5.531951, + "txtNewSaveName.Text": -6.320409, + "txtNewSaveName_OnActivate": -7.929846, + "very": -7.929846, + "walkable": -7.929846, + "while": -7.929846, + "will": -7.236699, + "you": -7.929846, + "your": -7.929846, + "{": -3.586041, + "||": -5.531951, + "}": -3.560399, + }, + "AMPL": map[string]float64{ + "(": -3.988984, + ")": -3.988984, + "*": -4.276666, + "+": -6.068426, + ",": -2.667228, + "-": -6.068426, + "..": -4.969813, + ":": -3.123987, + ";": -2.602690, + "<": -6.068426, + "=": -3.072693, + ">": -3.765840, + "BEAMS": -4.122515, + "COLUMNS": -4.276666, + "I": -4.122515, + "KnapsackBound": -4.969813, + "ROWS": -4.276666, + "S": -4.969813, + "T": -4.969813, + "Take": -4.969813, + "TotalValue": -6.068426, + "Value": -4.969813, + "Weight": -4.969813, + "WeightLimit": -6.068426, + "X": -4.458988, + "[": -3.178054, + "]": -3.178054, + "beam_values": -4.458988, + "binary": -6.068426, + "critical_area": -4.458988, + "critical_limit": -6.068426, + "critical_max": -5.375278, + "critical_values": -5.375278, + "data": -6.068426, + "h": -3.072693, + "i": -3.123987, + "in": -3.123987, + "integer": -5.375278, + "k": -3.072693, + "maximize": -5.375278, + "minimize": -4.969813, + "num_beams": -5.375278, + "num_cols": -5.375278, + "num_rows": -5.375278, + "param": -3.503476, + "s.t.": -6.068426, + "set": -4.276666, + "subject": -5.375278, + "sum": -3.765840, + "to": -5.375278, + "total_critical_dosage": -6.068426, + "total_critical_slack": -6.068426, + "total_tumor_dosage": -6.068426, + "total_tumor_slack": -6.068426, + "tumor_area": -4.458988, + "tumor_limit": -6.068426, + "tumor_min": -5.375278, + "tumor_values": -5.375278, + "var": -4.682131, + "{": -2.932931, + "}": -2.932931, + }, + "API Blueprint": map[string]float64{ + "!": -5.216746, + "#": -4.369448, + "##": -6.315358, + "%": -3.319626, + "'": -6.315358, + "(": -3.057261, + ")": -3.057261, + "**": -6.315358, + "**API": -6.315358, + "**Note": -6.315358, + "**parse**": -6.315358, + "**real": -6.315358, + "+": -3.096482, + ",": -3.542769, + "-": -3.917463, + ".": -5.622211, + "//github.com/apiaryio/drafter": -6.315358, + "//github.com/apiaryio/drafter#bindings": -6.315358, + "//raw.github.com/apiaryio/api": -4.929064, + ":": -2.760010, + "A": -4.523599, + "API": -4.118133, + "API.md": -5.622211, + "APIs": -6.315358, + "Action.md": -6.315358, + "Actions": -5.622211, + "Actions.md": -5.622211, + "Advanced": -4.705920, + "Also": -6.315358, + "As": -6.315358, + "Attributes": -5.216746, + "Attributes.md": -5.216746, + "Blueprint": -4.705920, + "Blueprint**": -6.315358, + "Blueprint**.": -6.315358, + "Body": -6.315358, + "Date": -6.315358, + "FORMAT": -5.216746, + "Github": -6.315358, + "Hello": -6.315358, + "ID.": -6.315358, + "In": -6.315358, + "JSON": -6.315358, + "Markdown": -6.315358, + "Model": -6.315358, + "Model.md": -6.315358, + "Next": -5.622211, + "One": -6.315358, + "Parameters": -4.929064, + "Parameters.md": -6.315358, + "Previous": -5.622211, + "Raw": -4.929064, + "Resource": -4.523599, + "Response": -4.705920, + "Retrieves": -6.315358, + "Simplest": -5.622211, + "The": -6.315358, + "This": -4.369448, + "Time": -6.315358, + "We": -6.315358, + "World": -6.315358, + "[": -3.750409, + "]": -3.750409, + "a": -4.369448, + "about": -6.315358, + "action": -5.622211, + "after": -6.315358, + "also": -6.315358, + "amount": -6.315358, + "an": -6.315358, + "and": -4.118133, + "another": -5.622211, + "application/json": -5.216746, + "apply": -6.315358, + "apply.": -6.315358, + "as": -5.622211, + "attributes": -5.622211, + "avoid": -6.315358, + "be": -6.315358, + "between": -6.315358, + "bindings": -6.315358, + "blueprint/master/examples/": -4.929064, + "body": -5.622211, + "can": -5.622211, + "case": -6.315358, + "code": -6.315358, + "combined": -6.315358, + "complementary": -6.315358, + "contains": -6.315358, + "coupon": -4.929064, + "course": -6.315358, + "created": -6.315358, + "customer.": -6.315358, + "demonstrate": -6.315358, + "demonstrates": -5.622211, + "describe": -6.315358, + "description": -6.315358, + "descriptions.": -6.315358, + "discount": -5.622211, + "do": -6.315358, + "duplicate": -6.315358, + "duplicates": -6.315358, + "every": -6.315358, + "example": -4.705920, + "examples": -6.315358, + "explain": -6.315358, + "fact": -6.315358, + "false": -5.622211, + "forget": -6.315358, + "given": -6.315358, + "going": -5.622211, + "how": -5.216746, + "https": -4.523599, + "id": -5.216746, + "in": -4.369448, + "information": -6.315358, + "installment": -6.315358, + "integer": -6.315358, + "is": -4.369448, + "it": -5.622211, + "its": -6.315358, + "just": -6.315358, + "keep": -6.315358, + "longer": -6.315358, + "message.": -6.315358, + "method": -6.315358, + "might": -6.315358, + "mind": -6.315358, + "next": -6.315358, + "no": -6.315358, + "not": -6.315358, + "null": -6.315358, + "number": -4.929064, + "object": -6.315358, + "of": -4.929064, + "off": -5.622211, + "on": -5.622211, + "one": -5.622211, + "opposed": -6.315358, + "or": -5.216746, + "output": -6.315358, + "parser": -6.315358, + "parser.": -6.315358, + "percent": -6.315358, + "percent_off": -6.315358, + "plain": -6.315358, + "please": -6.315358, + "positive": -6.315358, + "priority": -6.315358, + "progress": -6.315358, + "provided": -6.315358, + "really": -6.315358, + "redeem_by": -6.315358, + "redeemed": -6.315358, + "represents": -6.315358, + "request": -6.315358, + "resource": -5.216746, + "resource.": -6.315358, + "response": -6.315358, + "reuse": -6.315358, + "s": -6.315358, + "section.": -6.315358, + "see": -6.315358, + "seeing": -6.315358, + "simplest": -6.315358, + "single": -6.315358, + "stamp": -6.315358, + "state": -5.216746, + "status": -6.315358, + "string": -4.929064, + "such": -6.315358, + "text/plain": -6.315358, + "that": -5.216746, + "the": -3.482145, + "this": -5.622211, + "through": -6.315358, + "to": -3.917463, + "transition": -5.622211, + "transition.": -6.315358, + "true": -6.315358, + "view": -6.315358, + "want": -6.315358, + "we": -6.315358, + "what": -5.622211, + "which": -6.315358, + "will": -5.216746, + "with": -5.216746, + "written": -6.315358, + "you": -5.622211, + "{": -4.929064, + "}": -4.929064, + "–": -5.622211, + }, + "APL": map[string]float64{ + "!": -7.546446, + "#": -7.546446, + "#.": -6.853299, + "#.DISPLAY": -7.546446, + "#.Files.Dir": -7.546446, + "#.UT.appdir": -6.447834, + "#.UT.expect": -7.546446, + "#.UT.run": -7.546446, + "(": -3.521095, + ")": -3.503395, + "+": -5.061540, + ",": -3.080538, + "-": -5.349222, + "/": -4.907389, + "//tryapl.org/": -7.546446, + "/C": -7.546446, + "/Functions": -7.546446, + "/Page": -7.546446, + "/Z": -7.546446, + "/config": -7.546446, + "/input": -7.546446, + "/lines": -7.546446, + "/usr/local/bin/apl": -7.546446, + ":": -3.341754, + ";": -3.341754, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "": -6.853299, + "": -7.546446, + "": -7.546446, + "": -7.546446, + "
":              -7.546446,
+		"":           -8.083637,
+		"":           -5.598731,
+		"":           -8.083637,
+		"":              -4.751433,
+		"":            -6.291878,
+		"":           -6.985025,
+		"":              -5.444580,
+		"":                -8.083637,
+		"
":                -8.083637,
+		"":                -6.004196,
+		"":             -6.474199,
+		"
":              -6.985025,
+		"