feat(4300/3427): LSP integration in all flux monaco editors (#4566)

Our flux LSP integration with the monaco-editor was previously using the monaco-clientlanguage feature provider API, which required each feature to be wired together in code. Instead, we are now using a direct json-rpc message channel. Our implementation includes the LSP server hosted on a worker, and a fallback worker using a thin abstract layer to mimic the worker interface. We also need to provide the variable definitions (e.g. `v.windowPeriod`) to the LSP, within a separate prelude document.

A few more technical notes:
* the LSP currently assess documents in order. Therefore the prelude document must be created before the main editor document (monotonically increasing).
* variables are stored in state, in different locations based upon the application. Therefore the variables must be provided to the editor from the parent. 
* during hmr, the webpack worker plugin does not recall the namespace of the previously generated workers. Therefore, it may overwrite the csv worker with the LSP worker (e.g. both will be 0.worker.js). Hence the updates to webpack.
* the lsp server triggered requests, requires the onMessage method to be overwritten prior to the server run.

Author: wiedld

cypress/e2e/shared/editor.test.ts

@@ -0,0 +1,84 @@
+import {Organization} from '../../../src/types'
+
+// to see list of monaco-editor widgets to check:
+// document.querySelectorAll('[widgetid]')
+
+describe('Editor+LSP communication', () => {
+  const runTest = editorSelector => {
+    it('receives LSP-triggered server events', () => {
+      cy.getByTestID(editorSelector).then(() => {
+        cy.getByTestID('flux-editor', {timeout: 30000})
+          .should('be.visible')
+          .monacoType('foo |> bar')
+          .within(() => {
+            cy.get('.squiggly-error', {timeout: 30000}).should('be.visible')
+          })
+        cy.getByTestID('flux-editor').monacoType('{selectall}{del}')
+      })
+    })
+
+    it('has a roundtrip request-response, from the editor to the LSP server', () => {
+      cy.getByTestID(editorSelector).then(() => {
+        cy.getByTestID('flux-editor', {timeout: 30000})
+          .monacoType('{selectall} {backspace}')
+          .monacoType('from(')
+          .within(() => {
+            cy.get('[widgetid="editor.widget.suggestWidget"]', {
+              timeout: 30000,
+            }).should('be.visible')
+          })
+          .monacoType(`{selectall}{del}`)
+      })
+    })
+  }
+
+  describe('in Flows:', () => {
+    before(() => {
+      cy.flush()
+      cy.signin()
+      cy.get('@org').then(({id}: Organization) =>
+        cy.fixture('routes').then(({orgs}) => {
+          cy.visit(`${orgs}/${id}`)
+        })
+      )
+      cy.getByTestID('version-info')
+      cy.getByTestID('nav-item-flows').should('be.visible')
+      cy.getByTestID('nav-item-flows').click()
+
+      cy.getByTestID('preset-new')
+        .first()
+        .click()
+      cy.getByTestID('time-machine-submit-button').should('be.visible')
+
+      cy.get('.flow-divider--button')
+        .first()
+        .click()
+      cy.get('.insert-cell-menu.always-on').contains('Add Another Panel')
+      cy.getByTestID('add-flow-btn--rawFluxEditor')
+        .last()
+        .click()
+      cy.getByTestID('flux-editor').should('be.visible')
+    })
+
+    runTest('flux-editor')
+  })
+
+  describe('in DataExplorer', () => {
+    before(() => {
+      cy.flush()
+      cy.signin()
+      cy.get('@org').then(({id}: Organization) => {
+        cy.createMapVariable(id)
+        cy.fixture('routes').then(({orgs, explorer}) => {
+          cy.visit(`${orgs}/${id}${explorer}`)
+          cy.getByTestID('tree-nav').should('be.visible')
+        })
+      })
+      cy.getByTestID('switch-to-script-editor')
+        .should('be.visible')
+        .click()
+    })
+
+    runTest('time-machine--bottom')
+  })
+})

cypress/e2e/shared/explorer.test.ts

@@ -430,29 +430,16 @@ describe('DataExplorer', () => {
         .click()
     })
 
-    it('shows the proper errors and query button state', () => {
+    it('shows the proper query button state', () => {
       cy.getByTestID('time-machine-submit-button').should('be.disabled')
 
       cy.getByTestID('time-machine--bottom').then(() => {
         cy.getByTestID('flux-editor', {timeout: 30000})
           .should('be.visible')
-          .monacoType('foo |> bar')
-          .within(() => {
-            cy.get('.squiggly-error', {timeout: 30000}).should('be.visible')
-          })
-          .monacoType('{selectall} {backspace}')
-          .monacoType('from(')
-          .within(() => {
-            cy.get('[widgetid="editor.widget.suggestWidget"]', {
-              timeout: 30000,
-            }).should('be.visible')
-          })
-          .monacoType(`{selectall}{del}from(bucket: )`)
+          .monacoType(`{selectall}{del}from(bucket: "my-bucket")`)
       })
 
       cy.getByTestID('time-machine-submit-button').should('not.be.disabled')
-
-      cy.getByTestID('flux-editor').monacoType('yo')
       cy.getByTestID('flux-editor').monacoType('{selectall}{del}')
     })
 

package.json

@@ -162,6 +162,7 @@
     "webpack-merge": "^4.2.2"
   },
   "dependencies": {
+    "@codingame/monaco-jsonrpc": "^0.3.1",
     "@docsearch/react": "^3.0.0-alpha.37",
     "@influxdata/clockface": "^4.0.1",
     "@influxdata/flux-lsp-browser": "0.8.8",
@@ -220,6 +221,7 @@
     "rudder-sdk-js": "^1.0.16",
     "s2-geometry": "^1.2.10",
     "use-local-storage-state": "^9.0.2",
+    "vscode-languageserver-protocol": "^3.16.0",
     "worker-plugin": "^5.0.1",
     "y-websocket": "^1.4.3",
     "yjs": "^13.5.36"

src/dataExplorer/components/ResultsPane.tsx

@@ -23,6 +23,9 @@ import {downloadTextFile} from 'src/shared/utils/download'
 import {event} from 'src/cloud/utils/reporting'
 import {QueryContext} from 'src/shared/contexts/query'
 import {notify} from 'src/shared/actions/notifications'
+import {TIME_RANGE_START, TIME_RANGE_STOP} from 'src/variables/constants'
+import {getRangeVariable} from 'src/variables/utils/getTimeRangeVars'
+import {getWindowPeriodVariableFromVariables} from 'src/variables/utils/getWindowVars'
 
 import {DEFAULT_TIME_RANGE} from 'src/shared/constants/timeRanges'
 
@@ -98,6 +101,15 @@ const ResultsPane: FC = () => {
       })
   }
 
+  const timeVars = [
+    getRangeVariable(TIME_RANGE_START, timeRange),
+    getRangeVariable(TIME_RANGE_STOP, timeRange),
+  ]
+
+  const variables = timeVars.concat(
+    getWindowPeriodVariableFromVariables(text, timeVars) || []
+  )
+
   return (
     <DraggableResizer
       handleOrientation={Orientation.Horizontal}
@@ -120,7 +132,11 @@ const ResultsPane: FC = () => {
                 />
               }
             >
-              <FluxMonacoEditor script={text} onChangeScript={setText} />
+              <FluxMonacoEditor
+                variables={variables}
+                script={text}
+                onChangeScript={setText}
+              />
             </Suspense>
           </div>
           <div style={{width: '100%'}}>

src/flows/components/FlowPipe.tsx

@@ -3,6 +3,7 @@ import React, {FC, createElement, useMemo} from 'react'
 import Pipe from 'src/flows/components/Pipe'
 import FlowPanel from 'src/flows/components/panel/FlowPanel'
 import {PipeProvider} from 'src/flows/context/pipe'
+import {VariablesProvider} from 'src/flows/context/variables'
 
 export interface FlowPipeProps {
   id: string
@@ -12,15 +13,17 @@ const FlowPipe: FC<FlowPipeProps> = ({id}) => {
   return useMemo(
     () => (
       <PipeProvider id={id}>
-        <Pipe
-          Context={props => {
-            const _props = {
-              ...props,
-              id,
-            }
-            return createElement(FlowPanel, _props)
-          }}
-        />
+        <VariablesProvider>
+          <Pipe
+            Context={props => {
+              const _props = {
+                ...props,
+                id,
+              }
+              return createElement(FlowPanel, _props)
+            }}
+          />
+        </VariablesProvider>
       </PipeProvider>
     ),
     [id]

src/flows/components/QueryTextPreview.tsx

@@ -0,0 +1,49 @@
+import React, {FC, useContext, useEffect, useState} from 'react'
+import {Variable} from 'src/types'
+import {FlowQueryContext} from 'src/flows/context/flow.query'
+import {FlowContext} from 'src/flows/context/flow.current'
+import {PipeContext} from 'src/flows/context/pipe'
+import {getRangeVariable} from 'src/variables/utils/getTimeRangeVars'
+import {TIME_RANGE_START, TIME_RANGE_STOP} from 'src/variables/constants'
+import {getWindowPeriodVariableFromVariables} from 'src/variables/utils/getWindowVars'
+
+const EMPTY_STATE = [] as Variable[]
+
+interface VariablesContextType {
+  variables: Variable[]
+}
+
+const DEFAULT_CONTEXT: VariablesContextType = {
+  variables: [],
+}
+
+export const VariablesContext = React.createContext<VariablesContextType>(
+  DEFAULT_CONTEXT
+)
+
+export const VariablesProvider: FC = ({children}) => {
+  const {id, range: pipeRange} = useContext(PipeContext)
+  const {flow} = useContext(FlowContext)
+  const {getPanelQueries} = useContext(FlowQueryContext)
+  const {source} = getPanelQueries(id)
+  const [variables, setVariables] = useState(EMPTY_STATE)
+
+  useEffect(() => {
+    if ((!pipeRange && !flow?.range) || !source) {
+      return
+    }
+    const range = pipeRange || flow?.range
+    const timeVars = [
+      getRangeVariable(TIME_RANGE_START, range),
+      getRangeVariable(TIME_RANGE_STOP, range),
+    ]
+    const windowVar = getWindowPeriodVariableFromVariables(source, timeVars)
+    setVariables(!!windowVar ? timeVars.concat(windowVar) : timeVars)
+  }, [id, source, pipeRange, flow?.range])
+
+  return (
+    <VariablesContext.Provider value={{variables}}>
+      {children}
+    </VariablesContext.Provider>
+  )
+}

src/flows/context/variables.tsx

@@ -6,8 +6,9 @@ import {
   TechnoSpinner,
 } from '@influxdata/clockface'
 
-// Types
+// Types and Context
 import {PipeProp} from 'src/types/flows'
+import {VariablesContext} from 'src/flows/context/variables'
 
 // Components
 import {PipeContext} from 'src/flows/context/pipe'
@@ -23,6 +24,7 @@ const Query: FC<PipeProp> = ({Context}) => {
   const {data} = useContext(PipeContext)
   const {queries, activeQuery} = data
   const query = queries[activeQuery]
+  const {variables} = useContext(VariablesContext)
 
   return (
     <Context>
@@ -36,6 +38,7 @@ const Query: FC<PipeProp> = ({Context}) => {
       >
         <FluxMonacoEditor
           script={query.text}
+          variables={variables}
           onChangeScript={() => {}}
           onSubmitScript={() => {}}
           autogrow

src/flows/pipes/RawFluxEditor/readOnly.tsx

@@ -28,6 +28,7 @@ import {
   EditorProvider,
   InjectionType,
 } from 'src/flows/context/editor'
+import {VariablesContext} from 'src/flows/context/variables'
 
 // Components
 import SecretsList from 'src/flows/pipes/RawFluxEditor/SecretsList'
@@ -59,6 +60,7 @@ const Query: FC<PipeProp> = ({Context}) => {
   const {setEditor, inject, updateText} = editorContext
   const {queries, activeQuery} = data
   const query = queries[activeQuery]
+  const {variables} = useContext(VariablesContext)
 
   useEffect(() => {
     if (isFlagEnabled('fluxInjectSecrets')) {
@@ -155,6 +157,7 @@ const Query: FC<PipeProp> = ({Context}) => {
         >
           <FluxMonacoEditor
             script={query.text}
+            variables={variables}
             onChangeScript={updateText}
             setEditorInstance={setEditor}
             wrapLines="on"
@@ -163,7 +166,13 @@ const Query: FC<PipeProp> = ({Context}) => {
         </Suspense>
       </Context>
     ),
-    [RemoteDataState.Loading, query.text, updateText, editorContext.editor]
+    [
+      RemoteDataState.Loading,
+      query.text,
+      updateText,
+      editorContext.editor,
+      variables,
+    ]
   )
 }