From c12f1dec2bc99e282da4cd7d093d81a39e30b0ff Mon Sep 17 00:00:00 2001 From: Chris Wendt Date: Tue, 30 Apr 2019 21:47:06 -0700 Subject: [PATCH] Add tree-sitter for local j2d/refs --- package/package.json | 1 + package/src/api.ts | 2 +- package/src/handler.ts | 310 +++++++++++++++++++++++++++++++++++++ package/src/tree-sitter.js | 1 + package/tsconfig.json | 1 + package/yarn.lock | 32 ++++ template/tsconfig.json | 4 +- 7 files changed, 348 insertions(+), 3 deletions(-) create mode 120000 package/src/tree-sitter.js diff --git a/package/package.json b/package/package.json index 793c33b25..2bf53be68 100644 --- a/package/package.json +++ b/package/package.json @@ -58,6 +58,7 @@ "typescript": "^3.2.1" }, "dependencies": { + "babel-polyfill": "^6.26.0", "lodash": "^4.17.11", "rxjs": "^6.3.3", "sourcegraph": "^23.0.0" diff --git a/package/src/api.ts b/package/src/api.ts index 6b7d9dea1..e55a49e1c 100644 --- a/package/src/api.ts +++ b/package/src/api.ts @@ -108,7 +108,7 @@ export class API { vars: graphqlVars, sourcegraph: this.sourcegraph, }) - const results = [] + const results: any[] = [] for (const result of respObj.data.search.results.results) { if (result.symbols) { for (const sym of result.symbols) { diff --git a/package/src/handler.ts b/package/src/handler.ts index 0307bc600..d2e2bd164 100644 --- a/package/src/handler.ts +++ b/package/src/handler.ts @@ -8,6 +8,303 @@ import { TextDocument, Hover, } from 'sourcegraph' +import TreeSitter from './tree-sitter.js' + +namespace TreeSitter { + export type SyntaxNode = any + export type Point = any +} +TreeSitter.init() + .then(() => console.log('init')) + .catch(() => console.log('fail')) + +async function tsdef({ + text, + position, +}: { + text: string + position: Position +}): Promise { + const TypeScript = await TreeSitter.Language.load( + 'http://localhost:5001/tree-sitter-typescript.wasm' + ) + + function flatten(xs: any): T[] { + return [].concat.apply([], xs) + } + + function pretty(node) { + function prettyLines(node) { + return node + ? [ + node.type + + (node.namedChildren.length !== 0 + ? '' + : ' ' + node.text), + ...flatten( + node.namedChildren.map(x => + prettyLines(x).map(y => ' ' + y) + ) + ), + ] + : [] + } + return prettyLines(node).join('\n') + } + + type Selector = (node: TreeSitter.SyntaxNode) => TreeSitter.SyntaxNode[] + + function children(): Selector { + return node => node.namedChildren + } + + function nthchild(n: number): Selector { + return node => (node.namedChildren[n] ? [node.namedChildren[n]] : []) + } + + function pipe(...selectors: Selector[]): Selector { + return selectors.reduce( + (acc, cur) => n => flatten(acc(n).map(n => cur(n))), + x => [x] + ) + } + + function match(name: string): Selector { + return node => { + return node.type === name ? [node] : [] + } + } + + // function descendant(name: string): Selector { + // return node => [ + // ...flatten(node.namedChildren.map(n => descendant(name)(n))), + // ...(node.type === name ? [node] : []), + // ] + // } + + function choice(...selectors: Selector[]): Selector { + return node => { + for (const selector of selectors) { + const nodes = selector(node) + if (nodes.length > 0) { + return nodes + } + } + return [] + } + } + + const unimplemented = name => () => { + // console.log(name, 'is unimplemented') + return [] + } + + const selectorByLanguage = { + typescript: (() => { + const identifier = match('identifier') + const shorthand_property_identifier = match( + 'shorthand_property_identifier' + ) + // TODO figure out how to do more deeply nested object destructuring. Need laziness. + const pair = pipe( + match('pair'), + nthchild(1), + identifier + ) + const object = pipe( + match('object_pattern'), + children(), + choice(pair, shorthand_property_identifier) + ) + const array = unimplemented('array') + const rest_parameter = unimplemented('rest_parameter') + const optional_parameter = unimplemented('optional_parameter') + const _destructuring_pattern = choice(object, array) + const required_parameter = pipe( + match('required_parameter'), + children(), + choice(identifier, _destructuring_pattern) + ) + const formal_parameters = pipe( + match('formal_parameters'), + children(), + choice(required_parameter, rest_parameter, optional_parameter) + ) + const call_signature = pipe( + match('call_signature'), + children(), + formal_parameters + ) + const arrow_function = pipe( + match('arrow_function'), + nthchild(0), + choice(identifier, call_signature) + ) + const fn = pipe( + match('function'), + children(), + call_signature + ) + const variable_declarator = pipe( + match('variable_declarator'), + children(), + identifier + ) + const lexical_declaration = pipe( + match('lexical_declaration'), + children(), + variable_declarator + ) + const statement_block = pipe( + match('statement_block'), + children(), + lexical_declaration + ) + const program = pipe( + match('program'), + children(), + lexical_declaration + ) + return choice( + arrow_function, + fn, + statement_block, + program + // ...more + ) + })(), + } + + function definition({ + selector, + rootNode, + index, + position, + }: { + selector: Selector + rootNode: TreeSitter.SyntaxNode + index?: number + position?: TreeSitter.Point + }): TreeSitter.SyntaxNode | undefined { + const refNode = + (index && rootNode.descendantForIndex(index + 1)) || + rootNode.descendantForPosition(position) + if ( + !refNode || + !['identifier', 'shorthand_property_identifier'].includes( + refNode.type + ) + ) { + console.log('Node is not an identifier', refNode && refNode.type) + return undefined + } else { + const idenfitier = refNode.text + function go( + node: TreeSitter.SyntaxNode | null + ): TreeSitter.SyntaxNode | undefined { + if (node === null) { + return undefined + } else { + const hit = selector(node).find( + candidate => candidate.text === idenfitier + ) + return hit ? hit : go(node.parent) + } + } + return go(refNode) + } + } + + function test() { + function testCase({ + selector, + code, + }: { + selector: Selector + code: string + }) { + const parser = new TreeSitter() + parser.setLanguage(TypeScript) + const tree = parser.parse(code.replace('&', '').replace('*', '')) + + const defIndex = code.indexOf('&') + const refIndex = code.replace('&', '').indexOf('*') + + if (refIndex === -1) { + console.error('Test must contain *, but did not:', code) + } + + const def = definition({ + selector, + rootNode: tree.rootNode, + index: refIndex, + }) + + if (defIndex === -1 && def === undefined) { + return + } + + if (defIndex === -1 && def !== undefined) { + console.log('Found a definition when it should not have.') + return + } + + if (def === undefined) { + console.log('No def =(') + console.log(code) + console.log(pretty(tree.rootNode)) + return + } + + if (def.startIndex !== defIndex) { + console.log('Expected index', defIndex) + console.log('Got index', def.startIndex) + console.log(code) + console.log(pretty(tree.rootNode)) + } + } + + const testsByLanguage = { + typescript: [ + '() => *x', + '&x => *x', + '(&x) => *x', + '(&x, y) => *x', + '&x => y => *x', + 'function foo(&x) { function bar(y) { return *x + y } }', + 'function foo(&x, y) { return *x }', + '({&x}) => *x', + '({y:&x}) => *x', + '{const &x = 5; console.log(*x)}', + 'const &x = 5; console.log(*x)', + ], + } + + console.log('Testing...') + for (const key of Object.keys(testsByLanguage)) { + for (const code of testsByLanguage[key]) { + testCase({ selector: selectorByLanguage[key], code }) + } + } + console.log('Testing... done', new Date()) + } + + test() + + const parser = new TreeSitter() + parser.setLanguage(TypeScript) + const tree = parser.parse(text) + const d = definition({ + selector: selectorByLanguage['typescript'], + rootNode: tree.rootNode, + position: { row: position.line, column: position.character + 1 }, + }) + if (d) { + return d.startPosition + } else { + return undefined + } +} /** * identCharPattern is used to match identifier tokens @@ -700,6 +997,19 @@ export class Handler { return null } + const d = await tsdef({ + text: fileContent, + position: pos, + }) + if (d) { + return [ + new this.sourcegraph.Location( + new URL(doc.uri), + new this.sourcegraph.Position(d.row, d.column) + ), + ] + } + const tokenResult = findSearchToken({ text: fileContent, position: pos, diff --git a/package/src/tree-sitter.js b/package/src/tree-sitter.js new file mode 120000 index 000000000..dbfd3ea89 --- /dev/null +++ b/package/src/tree-sitter.js @@ -0,0 +1 @@ +../../../../tree-sitter/tree-sitter/target/release/tree-sitter.js \ No newline at end of file diff --git a/package/tsconfig.json b/package/tsconfig.json index 98895330a..83d9fe424 100644 --- a/package/tsconfig.json +++ b/package/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "node", "lib": ["es2018", "webworker"], "forceConsistentCasingInFileNames": true, + "noImplicitAny": false, "noErrorTruncation": true, "skipLibCheck": true, "declaration": false, diff --git a/package/yarn.lock b/package/yarn.lock index d240ecb1d..b8133bc8b 100644 --- a/package/yarn.lock +++ b/package/yarn.lock @@ -238,6 +238,23 @@ babel-code-frame@^6.22.0: esutils "^2.0.2" js-tokens "^3.0.2" +babel-polyfill@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz#379937abc67d7895970adc621f284cd966cf2153" + integrity sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM= + dependencies: + babel-runtime "^6.26.0" + core-js "^2.5.0" + regenerator-runtime "^0.10.5" + +babel-runtime@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" + integrity sha1-llxwWGaOgrVde/4E/yM3vItWR/4= + dependencies: + core-js "^2.4.0" + regenerator-runtime "^0.11.0" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -454,6 +471,11 @@ copy-descriptor@^0.1.0: resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= +core-js@^2.4.0, core-js@^2.5.0: + version "2.6.5" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.5.tgz#44bc8d249e7fb2ff5d00e0341a7ffb94fbf67895" + integrity sha512-klh/kDpwX8hryYL14M9w/xei6vrv6sE8gTHDG7/T/+SEovB/G4ejwcfE/CBzO6Edsu+OETZMZ3wcX/EjUkrl5A== + cross-spawn@^4: version "4.0.2" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-4.0.2.tgz#7b9247621c23adfdd3856004a823cbe397424d41" @@ -1813,6 +1835,16 @@ redent@^1.0.0: indent-string "^2.1.0" strip-indent "^1.0.1" +regenerator-runtime@^0.10.5: + version "0.10.5" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz#336c3efc1220adcedda2c9fab67b5a7955a33658" + integrity sha1-M2w+/BIgrc7dosn6tntaeVWjNlg= + +regenerator-runtime@^0.11.0: + version "0.11.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz#be05ad7f9bf7d22e056f9726cee5017fbf19e2e9" + integrity sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg== + regex-cache@^0.4.2: version "0.4.4" resolved "https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.4.tgz#75bdc58a2a1496cec48a12835bc54c8d562336dd" diff --git a/template/tsconfig.json b/template/tsconfig.json index e3be8baea..de4076e6f 100644 --- a/template/tsconfig.json +++ b/template/tsconfig.json @@ -8,10 +8,10 @@ "allowUnreachableCode": false, "allowUnusedLabels": false, "forceConsistentCasingInFileNames": true, - "noImplicitAny": true, + // "noImplicitAny": true, "noImplicitReturns": true, "noImplicitThis": true, - "noUnusedLocals": true, + // "noUnusedLocals": true, "noUnusedParameters": false, "strictFunctionTypes": true, "strictNullChecks": true,