Custom identCharPattern per language

languages.ts

@@ -299,6 +299,7 @@ export const languageSpecs: LanguageSpec[] = [
                 'watchr',
             ],
             commentStyle: shellStyle,
+            identCharPattern: /[A-Za-z0-9_!?]/,
         },
         stylized: 'Ruby',
     },
@@ -379,6 +380,7 @@ export const languageSpecs: LanguageSpec[] = [
                 ...pythonStyle,
                 docPlacement: 'above the definition',
             },
+            identCharPattern: /[A-Za-z0-9_!?]/,
         },
         stylized: 'Elixir',
     },
@@ -421,6 +423,7 @@ export const languageSpecs: LanguageSpec[] = [
             languageID: 'clojure',
             fileExts: ['clj', 'cljs', 'cljx'],
             commentStyle: lispStyle,
+            identCharPattern: /[A-Za-z0-9_\-!?]/,
         },
         stylized: 'Clojure',
     },
@@ -436,6 +439,7 @@ export const languageSpecs: LanguageSpec[] = [
                     endRegex: /-}/,
                 },
             },
+            identCharPattern: /[A-Za-z0-9_']/,
         },
         stylized: 'Haskell',
     },
@@ -451,6 +455,7 @@ export const languageSpecs: LanguageSpec[] = [
                     endRegex: /#>/,
                 },
             },
+            identCharPattern: /[A-Za-z0-9_?]/,
         },
         stylized: 'PowerShell',
     },
@@ -469,6 +474,7 @@ export const languageSpecs: LanguageSpec[] = [
                 'el',
             ],
             commentStyle: lispStyle,
+            identCharPattern: /[A-Za-z0-9_!?]/,
         },
         stylized: 'Lisp',
     },
@@ -519,6 +525,7 @@ export const languageSpecs: LanguageSpec[] = [
             languageID: 'r',
             fileExts: ['r', 'R', 'rd', 'rsx'],
             commentStyle: { lineRegex: /#'?\s?/ },
+            identCharPattern: /[A-Za-z0-9_\.]/,
         },
         stylized: 'R',
     },

package/src/handler.test.ts

@@ -4,6 +4,7 @@ import {
     definitionQueries,
     findDocstring,
     wrapIndentationInCodeBlocks,
+    findSearchToken,
 } from './handler'
 import { TextDocument } from 'sourcegraph'
 import { pythonStyle, cStyle } from '../../languages'
@@ -301,3 +302,19 @@ prose
         )
     })
 })
+
+describe('findSearchToken', () => {
+    it('custom identCharPattern', () => {
+        assert.deepStrictEqual(
+            findSearchToken({
+                text: '(defn skip-ws! []',
+                position: { line: 0, character: 6 },
+                identCharPattern: /[A-Za-z0-9_\-!?]/,
+            }),
+            {
+                isComment: false,
+                searchToken: 'skip-ws!',
+            }
+        )
+    })
+})

package/src/handler.ts

@@ -10,9 +10,11 @@ import {
 } from 'sourcegraph'
 
 /**
- * identCharPattern is used to match identifier tokens
+ * The default regex for characters allowed in an identifier. It works well for
+ * C-like languages (C/C++, C#, Java, etc.) but not for languages that allow
+ * punctuation characters (e.g. Ruby).
  */
-const identCharPattern = /[A-Za-z0-9_\-']/
+const DEFAULT_IDENT_CHAR_PATTERN = /[A-Za-z0-9_]/
 
 /**
  * Selects documents that the extension works on.
@@ -122,15 +124,18 @@ function resultToLocation({
     }
 }
 
-function findSearchToken({
+export function findSearchToken({
     text,
     position,
     lineRegex,
+    identCharPattern,
 }: {
     text: string
-    position: Position
+    position: { line: number; character: number }
     lineRegex?: RegExp
+    identCharPattern?: RegExp
 }): { searchToken: string; isComment: boolean } | undefined {
+    identCharPattern = identCharPattern || DEFAULT_IDENT_CHAR_PATTERN
     const lines = text.split('\n')
     const line = lines[position.line]
     let end = line.length
@@ -572,6 +577,10 @@ export interface HandlerArgs {
      */
     docstringIgnore?: RegExp
     commentStyle?: CommentStyle
+    /**
+     * Regex that matches characters in an identifier.
+     */
+    identCharPattern?: RegExp
     sourcegraph: typeof import('sourcegraph')
     /**
      * Callback that filters the given symbol search results (e.g. to drop
@@ -589,6 +598,7 @@ export class Handler {
     public languageID: string = ''
     public fileExts: string[] = []
     public commentStyle: CommentStyle | undefined
+    public identCharPattern: RegExp | undefined
     public docstringIgnore: RegExp | undefined
     public debugAnnotatedURIs: string[]
     public filterDefinitions: FilterDefinitions
@@ -601,6 +611,7 @@ export class Handler {
         languageID,
         fileExts = [],
         commentStyle,
+        identCharPattern,
         docstringIgnore,
         sourcegraph,
         filterDefinitions: filterDefinitions = ({ results }) => results,
@@ -610,6 +621,7 @@ export class Handler {
         this.languageID = languageID
         this.fileExts = fileExts
         this.commentStyle = commentStyle
+        this.identCharPattern = identCharPattern
         this.docstringIgnore = docstringIgnore
         this.debugAnnotatedURIs = []
         this.filterDefinitions = filterDefinitions
@@ -704,6 +716,7 @@ export class Handler {
             text: fileContent,
             position: pos,
             lineRegex: this.commentStyle && this.commentStyle.lineRegex,
+            identCharPattern: this.identCharPattern,
         })
         if (!tokenResult) {
             return null
@@ -757,6 +770,7 @@ export class Handler {
             text: doc.text,
             position: pos,
             lineRegex: this.commentStyle && this.commentStyle.lineRegex,
+            identCharPattern: this.identCharPattern,
         })
         if (!tokenResult) {
             return []