Merge pull request #107514 from Profpatsch/tree-sitter-fetch-all-gram…

…mars

Tree sitter fetch all grammars

pkgs/development/tools/parsing/tree-sitter/default.nix

@@ -1,6 +1,6 @@
 { lib, stdenv
 , fetchgit, fetchFromGitHub, fetchurl
-, writeShellScript, runCommand, which
+, writeShellScript, runCommand, which, formats
 , rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten
 , Security
 , callPackage
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats lib;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
@@ -43,14 +43,22 @@ let
 
   builtGrammars = let
     change = name: grammar:
-      callPackage ./library.nix {} {
+      callPackage ./grammar.nix {} {
         language = name;
         inherit version;
         source = fetchGrammar grammar;
       };
   in
-    # typescript doesn't have parser.c in the same place as others
-    lib.mapAttrs change (removeAttrs (import ./grammars) ["typescript"]);
+    lib.mapAttrs change (removeAttrs (import ./grammars) [
+      # TODO these don't have parser.c in the same place as others.
+      # They might require more elaborate builds?
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-typescript"
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-ocaml"
+      # /nix/…/src/parser.c:1:10: fatal error: tree_sitter/parser.h: No such file or directory
+      "tree-sitter-razor"
+    ]);
 
 in rustPlatform.buildRustPackage {
   pname = "tree-sitter";
@@ -93,6 +101,11 @@ in rustPlatform.buildRustPackage {
     };
     inherit grammars;
     inherit builtGrammars;
+
+    tests = {
+      # make sure all grammars build
+      builtGrammars = lib.recurseIntoAttrs builtGrammars;
+    };
   };
 
   meta = {

pkgs/development/tools/parsing/tree-sitter/library.nix → pkgs/development/tools/parsing/tree-sitter/grammar.nix

@@ -15,7 +15,7 @@
 
 stdenv.mkDerivation {
 
-  pname = "tree-sitter-${language}-library";
+  pname = "${language}-grammar";
   inherit version;
 
   src = source;

pkgs/development/tools/parsing/tree-sitter/grammars/default.nix

@@ -1,18 +1,31 @@
 {
-  bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
-  go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
-  jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
-  lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
-  php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
-  python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
+  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
+  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
+  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
+  tree-sitter-lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
 }

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-agda",
+  "rev": "d710ff14d15ddee3764fd73a0837c6c4c8c913e9",
+  "date": "2019-09-20T18:06:06+08:00",
+  "path": "/nix/store/wqz9v9znaiwhhqi19hgig9bn0yvl4i9s-tree-sitter-agda",
+  "sha256": "1wpfj47l97pxk3i9rzdylqipy849r482fnj3lmx8byhalv7z1vm6",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-c-sharp",
+  "rev": "b8bff2a389ecc098dbb7e8abee492816a5eb42db",
+  "date": "2020-11-15T07:54:17+00:00",
+  "path": "/nix/store/h5p71g661hbyzcdcj6xff8y5pcsivpa4-tree-sitter-c-sharp",
+  "sha256": "0x78s2wgd8b6pwjzbmc9fgp0ivdmxv39wikig1m55slai6yq51wh",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-cpp",
-  "rev": "fb8250eef8b4cf1ce104806c50dc206e388b0e72",
-  "date": "2020-11-02T09:46:58-08:00",
-  "path": "/nix/store/qmza43f0fraa111bg50vaxsgnrs5kqs1-tree-sitter-cpp",
-  "sha256": "1sbk5a2p1fh6798naiswsap6fpj7n7bqhwd3xdyi35xf613qpwsi",
+  "rev": "a35a275df92e7583df38f2de2562361f2b69987e",
+  "date": "2020-12-13T11:27:21-08:00",
+  "path": "/nix/store/l0mv4q1xdxz94ym1nl73y52i1yr9zcgi-tree-sitter-cpp",
+  "sha256": "130vizybkm11j3lpzmf183myz0vjxq75mpy6qz48rrkidhnrlryk",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-css",
+  "rev": "23f2cb97d47860c517f67f03e1f4b621d5bd2085",
+  "date": "2020-05-14T14:44:30-07:00",
+  "path": "/nix/store/r5pkz9kly0mhgrmqzdzdsr6d1dpqavld-tree-sitter-css",
+  "sha256": "17svpf36p0p7spppzhm3fi833zpdl2l1scg34r6d4vcbv7dknrjy",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-fluent",
+  "rev": "858fdd6f1e81992e00d3541bfb31bac9365d7a47",
+  "date": "2018-06-18T13:00:38-07:00",
+  "path": "/nix/store/zbj8abdlrqi9swm8qn8rhpqmjwcz145f-tree-sitter-fluent",
+  "sha256": "0528v9w0cs73p9048xrddb1wpdhr92sn1sw8yyqfrq5sq0danr9k",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-haskell",
+  "rev": "2a0aa1cb5f1b787a4056a29fa0791e87846e33fb",
+  "date": "2018-11-03T09:56:20-07:00",
+  "path": "/nix/store/9xszs4xi51qr72laxr67zxnh8y2br0gy-tree-sitter-haskell",
+  "sha256": "0z0nfip5m0yrjgm49j5nld7jkgyjdmps101xhbng39pwfnh1av83",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-javascript",
-  "rev": "852f11b394804ac2a8986f8bcaafe77753635667",
-  "date": "2020-10-27T13:43:40-04:00",
-  "path": "/nix/store/7a9nd3prxbv2izvilqdxf58his097ak7-tree-sitter-javascript",
-  "sha256": "1cdqx75fm7fvna0iymw7n03a7f6gayfg97qwqz3himi0775fz9ir",
+  "rev": "3f8b62f9befd3cb3b4cb0de22f6595a0aadf76ca",
+  "date": "2020-12-02T10:20:20-08:00",
+  "path": "/nix/store/c17bf7sjq95lank5ygbglv8j48i5z9w3-tree-sitter-javascript",
+  "sha256": "0fjq1jzrzd8c8rfxkh2s25gnqlyc19k3a8i3r1129kakisn1288k",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-julia",
+  "rev": "6a0863f1ce3fcf6f99dc0addb7886dcbd27c5a48",
+  "date": "2020-09-08T19:39:52-07:00",
+  "path": "/nix/store/xn5nii9mi2aw7xdabyxlglha2vk12h1w-tree-sitter-julia",
+  "sha256": "07ds4wzgvnkgkq07izdglkk8lgpqrylvrs96afnwxi56bnzs8sbv",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/nvim-treesitter/tree-sitter-lua",
-  "rev": "278b0ea1f1f3d86afc86faaca0cbbf6c01182a54",
-  "date": "2020-09-08T18:29:23+02:00",
-  "path": "/nix/store/xkaqsw030bf6zd6pivrzrg3d50j1ka1k-tree-sitter-lua",
-  "sha256": "0i8jdp0bbl02h58hfhpild4v3rvy8yk5r5l90navvfxw4ad4f6f3",
+  "rev": "97b757ad3546d68dc1131e6ffb9b1d2a750eea29",
+  "date": "2020-11-25T21:23:57+01:00",
+  "path": "/nix/store/irz0dikk9dahkayi41chcznqq1i3wr84-tree-sitter-lua",
+  "sha256": "1nhg139vk3xm6ip4mhr29z0rprfg4q417z2vlvkz3m9wp8gppzls",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ocaml",
+  "rev": "4ec9ee414dadc2b6e7325a9f8124d02f6cd8c250",
+  "date": "2020-09-18T02:13:40+02:00",
+  "path": "/nix/store/dszjdnwnbziqxav2khs85026msm6fasz-tree-sitter-ocaml",
+  "sha256": "0wy85940fhmrnz7c1gk6xkipm8ixzalq5q4i7mcc6wnjiiwq60gx",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-python",
-  "rev": "d245768132eb6cb74d8a394ca4d29dc57169b096",
-  "date": "2020-10-27T13:20:04-04:00",
-  "path": "/nix/store/afv7ibxpf5c2shafa89cnza88hsfb7kj-tree-sitter-python",
-  "sha256": "075r5i2id2rn76xm3pcrn5cpvj63dlaxcpvphig39a4c9f9hrpdx",
+  "rev": "65b486b42fe0188e4be344092151042bf1bc6d9a",
+  "date": "2020-12-15T09:33:20-08:00",
+  "path": "/nix/store/k9jziqzyxq2bv55pwl03jcmmca83fjyp-tree-sitter-python",
+  "sha256": "1yxqdlmp0jybm7vvza1ni5a320vrviqkd14pnpcrg9ilzq23mlsh",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ql",
+  "rev": "a0d688d62dcb9cbc7c53f0d98343c458b3776b3d",
+  "date": "2020-09-16T12:56:09-07:00",
+  "path": "/nix/store/dfdaf6wg80dfw5fvdiir7n9nj6j30g3g-tree-sitter-ql",
+  "sha256": "0f6rfhrbvpg8czfa7mld45by3rp628bs6fyl47a8mn18w6x0n5g2",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-razor",
+  "rev": "60edbd8e798e416f5226a746396efa6a8614fa9b",
+  "date": "2016-07-08T15:17:50-07:00",
+  "path": "/nix/store/2vkxw48wc0xf07awi65r4rg2m1lpqq6c-tree-sitter-razor",
+  "sha256": "1ldsn9nxldxkxdy7irywx88d6a56q05f6907lypzass07piyqp2i",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-regex",
+  "rev": "be2e415b5716615530234d179dc27c32b7a1d86b",
+  "date": "2020-05-14T15:05:54-07:00",
+  "path": "/nix/store/acbcq9lgh5pf6hq3329fa253yfvaj6ji-tree-sitter-regex",
+  "sha256": "0qxbafbwfj64bwhj3455jgkidpdzqwrlsqm3rckq2pi75flnkv42",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-ruby",
-  "rev": "f86d90cce578e28c824afdcdbdca7d031b88c0b0",
-  "date": "2020-11-02T20:49:49-08:00",
-  "path": "/nix/store/gs0hhb1dlgmv0y60zyyz7ki30yvpddfx-tree-sitter-ruby",
-  "sha256": "1n1irdjfmv27p5ia2vqzv8zfsq19pxi0as0wrkb7z7s1lqr1gwrb",
+  "rev": "bb572f60e9538bd11fbde95a54f97522073f1e06",
+  "date": "2020-12-20T12:25:50-08:00",
+  "path": "/nix/store/hkjgvxh8l4drk1z4zkhmd0wcj876x5y1-tree-sitter-ruby",
+  "sha256": "198sfmrb3mxvpfmlwmwfmb3qs075ih0bjmr0ffrdy4nqm4a00cb8",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-rust",
-  "rev": "cf47ff80cbd8bdc6dd649a005c5f5d73f9c347e2",
-  "date": "2020-11-04T06:23:40-08:00",
-  "path": "/nix/store/8cvjxw8cz5jrkqwvgq88r7rfm269xwk1-tree-sitter-rust",
-  "sha256": "0yzcag0yy3ncxwpnb3fmsw8bxaidp7z8kbl8wwaxaj2vvxxf6sam",
+  "rev": "8746bd4b584b8063ee8e445bf31015e887417d33",
+  "date": "2020-11-23T17:16:42-08:00",
+  "path": "/nix/store/1zzxvza23wsdyazw47lhjvrs0za6wjpn-tree-sitter-rust",
+  "sha256": "0bhxfyq8ycnp90pqvr6cf3gpq1vax8a34kaq85dmbrc6ar8a7ap6",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-scala",
+  "rev": "211bb726bb5857f872247b600c7c1808e641a8d4",
+  "date": "2020-07-13T13:31:00-07:00",
+  "path": "/nix/store/6q66gzabxazr2581dgp4pp5wwyk2p6mw-tree-sitter-scala",
+  "sha256": "096ps5za8gxmq61gdd3xdk8cif07vb4v8asls2kdwm6jazm82777",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-swift",
+  "rev": "a22fa5e19bae50098e2252ea96cba3aba43f4c58",
+  "date": "2019-10-24T19:04:02-06:00",
+  "path": "/nix/store/pk5xk8yp6vanbar75bhfrs104w0k1ph0-tree-sitter-swift",
+  "sha256": "14b40lmwrnyvdz2wiv684kfh4fvqfhbj1dgrx81ppmy7hsz7jcq7",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json

@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-typescript",
-  "rev": "73afadbd117a8e8551758af9c3a522ef46452119",
-  "date": "2020-10-20T12:15:52-07:00",
-  "path": "/nix/store/wrgp1j5l50cigv5cmlxikw693a55i6g9-tree-sitter-typescript",
-  "sha256": "0dsbmcvjzys9s229drp1l8anram7d6nsx1nixl5m9znw54hr4w9p",
+  "rev": "a3a4becef889692724bb1c2191f8fc4bce6573f9",
+  "date": "2020-12-15T13:34:55-08:00",
+  "path": "/nix/store/dra5wbhgxkkhphm1mhgv87hsfw0g18nw-tree-sitter-typescript",
+  "sha256": "0ir7lmzwnbf1zwbl4bfsib1jilg8pvfc1nabaq2n2rixvghlvf3d",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json

@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-verilog",
+  "rev": "ad551aae2649da56582bc0557478a7dc979c0be3",
+  "date": "2020-10-13T17:40:47-07:00",
+  "path": "/nix/store/nfaxfqrqkxpwaq8rnk7kcp28nnj8y6m2-tree-sitter-verilog",
+  "sha256": "0cy29i200rnc34d237s19r6a1n5vv4d3wgwpbywxg6ahcankc34m",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}