Extend and make glob pattern configurable

server/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Bash Language Server
 
+## 1.8.0
+
+* Extend file glob used for pre-analyzing files from `**/*.sh` to `**/*@(.sh|.inc|.bash|.command)`
+* Make file glob configurable with `GLOB_PATTERN` environment variable
+
 ## 1.7.0
 
 * Add PATH tilde expansion

server/package.json

@@ -3,7 +3,7 @@
   "description": "A language server for Bash",
   "author": "Mads Hartmann",
   "license": "MIT",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "publisher": "mads-hartmann",
   "main": "./out/server.js",
   "typings": "./out/server.d.ts",
@@ -18,7 +18,7 @@
     "node": "*"
   },
   "dependencies": {
-    "glob": "^7.1.2",
+    "glob": "^7.1.6",
     "request": "^2.83.0",
     "request-promise-native": "^1.0.5",
     "turndown": "^4.0.2",

server/src/__tests__/analyzer.test.ts

@@ -1,4 +1,4 @@
-import FIXTURES from '../../../testing/fixtures'
+import FIXTURES, { FIXTURE_FOLDER } from '../../../testing/fixtures'
 import Analyzer from '../analyser'
 import { initializeParser } from '../parser'
 
@@ -97,3 +97,39 @@ describe('findSymbolCompletions', () => {
     expect(analyzer.findSymbolCompletions(CURRENT_URI)).toMatchSnapshot()
   })
 })
+
+describe('fromRoot', () => {
+  it('initializes an analyzer from a root', async () => {
+    const parser = await initializeParser()
+
+    jest.spyOn(Date, 'now').mockImplementation(() => 0)
+
+    const connection: any = {
+      console: {
+        log: jest.fn(),
+      },
+    }
+
+    const newAnalyzer = await Analyzer.fromRoot({
+      connection,
+      rootPath: FIXTURE_FOLDER,
+      parser,
+    })
+
+    expect(newAnalyzer).toBeDefined()
+
+    const FIXTURE_FILES_MATCHING_GLOB = 8
+    const LOG_LINES = FIXTURE_FILES_MATCHING_GLOB + 3
+
+    expect(connection.console.log).toHaveBeenCalledTimes(LOG_LINES)
+    expect(connection.console.log).toHaveBeenNthCalledWith(
+      1,
+      expect.stringContaining('Analyzing files matching'),
+    )
+
+    expect(connection.console.log).toHaveBeenNthCalledWith(
+      LOG_LINES,
+      'Analyzer finished after 0 seconds',
+    )
+  })
+})

server/src/__tests__/config.test.ts

@@ -7,6 +7,14 @@ describe('getExplainshellEndpoint', () => {
     expect(result).toBeNull()
   })
 
+  it('default to null in case of an empty string', () => {
+    process.env = {
+      EXPLAINSHELL_ENDPOINT: '',
+    }
+    const result = config.getExplainshellEndpoint()
+    expect(result).toBeNull()
+  })
+
   it('parses environment variable', () => {
     process.env = {
       EXPLAINSHELL_ENDPOINT: 'localhost:8080',
@@ -16,6 +24,30 @@ describe('getExplainshellEndpoint', () => {
   })
 })
 
+describe('getGlobPattern', () => {
+  it('default to a basic glob', () => {
+    process.env = {}
+    const result = config.getGlobPattern()
+    expect(result).toEqual(config.DEFAULT_GLOB_PATTERN)
+  })
+
+  it('default to a basic glob in case of an empty string', () => {
+    process.env = {
+      GLOB_PATTERN: '',
+    }
+    const result = config.getGlobPattern()
+    expect(result).toEqual(config.DEFAULT_GLOB_PATTERN)
+  })
+
+  it('parses environment variable', () => {
+    process.env = {
+      GLOB_PATTERN: '*.*',
+    }
+    const result = config.getGlobPattern()
+    expect(result).toEqual('*.*')
+  })
+})
+
 describe('highlightParsingError', () => {
   it('default to true', () => {
     process.env = {}

server/src/analyser.ts

@@ -1,13 +1,13 @@
 import * as fs from 'fs'
-import * as glob from 'glob'
-import * as Path from 'path'
 import * as request from 'request-promise-native'
 import * as URI from 'urijs'
 import * as LSP from 'vscode-languageserver'
 import * as Parser from 'web-tree-sitter'
 
+import { getGlobPattern } from './config'
 import { uniqueBasedOnHash } from './util/array'
 import { flattenArray, flattenObjectValues } from './util/flatten'
+import { getFilePaths } from './util/fs'
 import * as TreeSitterUtil from './util/tree-sitter'
 
 type Kinds = { [type: string]: LSP.SymbolKind }
@@ -27,10 +27,10 @@ export default class Analyzer {
    * Initialize the Analyzer based on a connection to the client and an optional
    * root path.
    *
-   * If the rootPath is provided it will initialize all *.sh files it can find
-   * anywhere on that path.
+   * If the rootPath is provided it will initialize all shell files it can find
+   * anywhere on that path. This non-exhaustive glob is used to preload the parser.
    */
-  public static fromRoot({
+  public static async fromRoot({
     connection,
     rootPath,
     parser,
@@ -39,39 +39,37 @@ export default class Analyzer {
     rootPath: string | null
     parser: Parser
   }): Promise<Analyzer> {
-    // This happens if the users opens a single bash script without having the
-    // 'window' associated with a specific project.
-    if (!rootPath) {
-      return Promise.resolve(new Analyzer(parser))
-    }
+    const analyzer = new Analyzer(parser)
+
+    if (rootPath) {
+      const globPattern = getGlobPattern()
+      connection.console.log(
+        `Analyzing files matching glob "${globPattern}" inside ${rootPath}`,
+      )
 
-    return new Promise((resolve, reject) => {
-      glob('**/*.sh', { cwd: rootPath }, (err, paths) => {
-        if (err != null) {
-          reject(err)
-        } else {
-          const analyzer = new Analyzer(parser)
-          paths.forEach(p => {
-            const absolute = Path.join(rootPath, p)
-            // only analyze files, glob pattern may match directories
-            if (fs.existsSync(absolute) && fs.lstatSync(absolute).isFile()) {
-              const uri = `file://${absolute}`
-              connection.console.log(`Analyzing ${uri}`)
-              analyzer.analyze(
-                uri,
-                LSP.TextDocument.create(
-                  uri,
-                  'shell',
-                  1,
-                  fs.readFileSync(absolute, 'utf8'),
-                ),
-              )
-            }
-          })
-          resolve(analyzer)
-        }
+      const lookupStartTime = Date.now()
+      const getTimePassed = (): string =>
+        `${(Date.now() - lookupStartTime) / 1000} seconds`
+
+      // NOTE: An alternative would be to preload all files and analyze their
+      // shebang or mimetype, but it would be fairly expensive.
+      const filePaths = await getFilePaths({ globPattern, rootPath })
+
+      connection.console.log(
+        `Glob resolved with ${filePaths.length} files after ${getTimePassed()}`,
+      )
+
+      filePaths.forEach(filePath => {
+        const uri = `file://${filePath}`
+        connection.console.log(`Analyzing ${uri}`)
+        const fileContent = fs.readFileSync(filePath, 'utf8')
+        analyzer.analyze(uri, LSP.TextDocument.create(uri, 'shell', 1, fileContent))
       })
-    })
+
+      connection.console.log(`Analyzer finished after ${getTimePassed()}`)
+    }
+
+    return analyzer
   }
 
   private parser: Parser

server/src/config.ts

@@ -1,6 +1,17 @@
+export const DEFAULT_GLOB_PATTERN = '**/*@(.sh|.inc|.bash|.command)'
+
 export function getExplainshellEndpoint(): string | null {
   const { EXPLAINSHELL_ENDPOINT } = process.env
-  return typeof EXPLAINSHELL_ENDPOINT !== 'undefined' ? EXPLAINSHELL_ENDPOINT : null
+  return typeof EXPLAINSHELL_ENDPOINT === 'string' && EXPLAINSHELL_ENDPOINT.trim() !== ''
+    ? EXPLAINSHELL_ENDPOINT
+    : null
+}
+
+export function getGlobPattern(): string {
+  const { GLOB_PATTERN } = process.env
+  return typeof GLOB_PATTERN === 'string' && GLOB_PATTERN.trim() !== ''
+    ? GLOB_PATTERN
+    : DEFAULT_GLOB_PATTERN
 }
 
 export function getHighlightParsingError(): boolean {

server/src/util/fs.ts

@@ -1,4 +1,5 @@
 import * as Fs from 'fs'
+import * as glob from 'glob'
 import * as Os from 'os'
 
 export function getStats(path: string): Promise<Fs.Stats> {
@@ -20,3 +21,24 @@ export function untildify(pathWithTilde: string): string {
     ? pathWithTilde.replace(/^~(?=$|\/|\\)/, homeDirectory)
     : pathWithTilde
 }
+
+export async function getFilePaths({
+  globPattern,
+  rootPath,
+}: {
+  globPattern: string
+  rootPath: string
+}): Promise<string[]> {
+  return new Promise((resolve, reject) => {
+    glob(globPattern, { cwd: rootPath, nodir: true, absolute: true }, function(
+      err,
+      files,
+    ) {
+      if (err) {
+        return reject(err)
+      }
+
+      resolve(files)
+    })
+  })
+}

server/yarn.lock

@@ -328,10 +328,10 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-glob@^7.1.2:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
-  integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==
+glob@^7.1.6:
+  version "7.1.6"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
+  integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
   dependencies:
     fs.realpath "^1.0.0"
     inflight "^1.0.4"

testing/fixtures.ts

@@ -2,14 +2,14 @@ import * as fs from 'fs'
 import * as path from 'path'
 import * as LSP from 'vscode-languageserver'
 
-const base = path.join(__dirname, './fixtures/')
+export const FIXTURE_FOLDER = path.join(__dirname, './fixtures/')
 
 function getFixture(filename: string) {
   return LSP.TextDocument.create(
     'foo',
     'bar',
     0,
-    fs.readFileSync(path.join(base, filename), 'utf8'),
+    fs.readFileSync(path.join(FIXTURE_FOLDER, filename), 'utf8'),
   )
 }
 

testing/fixtures/extension

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "It works, but is not parsed initially"

testing/fixtures/extension.inc

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+RED=`tput setaf 1`
+GREEN=`tput setaf 2`
+BLUE=`tput setaf 4`
+BOLD=`tput bold`
+RESET=`tput sgr0`

testing/fixtures/not-a-shell-script.sh

@@ -0,0 +1,4 @@
+if this is not a shell script, then it should not be parsed
+
+echo "Or is it?"
+

testing/fixtures/not-a-shell-script2.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env python2.7
+# set -x
+
+def func():
+  print 'hello world'

testing/fixtures/sourcing.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+source ./extension.inc
+
+echo $RED 'Hello in red!'

vscode-client/package.json

@@ -31,10 +31,15 @@
       "type": "object",
       "title": "Bash IDE configuration",
       "properties": {
+        "bashIde.globPattern": {
+          "type": "string",
+          "default": "**/*@(.sh|.inc|.bash|.command)",
+          "description": "Glob pattern for finding and parsing shell script files."
+        },
         "bashIde.highlightParsingErrors": {
           "type": "boolean",
           "default": true,
-          "description": "If enabled parsing errors will be highlighted as 'problems' "
+          "description": "If enabled parsing errors will be highlighted as problems."
         },
         "bashIde.explainshellEndpoint": {
           "type": "string",

vscode-client/src/extension.ts

@@ -13,13 +13,16 @@ export async function activate(context: ExtensionContext) {
     .getConfiguration('bashIde')
     .get('explainshellEndpoint', '')
 
+  const globPattern = workspace.getConfiguration('bashIde').get('globPattern', '')
+
   const highlightParsingErrors = workspace
     .getConfiguration('bashIde')
     .get('highlightParsingErrors', false)
 
   const env: any = {
     ...process.env,
     EXPLAINSHELL_ENDPOINT: explainshellEndpoint,
+    GLOB_PATTERN: globPattern,
     HIGHLIGHT_PARSING_ERRORS: highlightParsingErrors,
   }