From 9d611d09eec2616f226b53b6af9c707623159f6c Mon Sep 17 00:00:00 2001 From: cccs-jc Date: Fri, 27 Aug 2021 10:27:03 -0400 Subject: [PATCH] added json adapter additional adapter to load schema from a file monitor file for changes and reload schema preparation work for functions --- .gitignore | 1 + packages/server/src/SettingStore.ts | 10 +- packages/server/src/complete.ts | 14 +-- packages/server/src/createServer.ts | 110 +++++++++++++----- .../src/database_libs/AbstractClient.ts | 17 ++- packages/server/test/complete.test.ts | 10 +- 6 files changed, 116 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 518b5472..31deb5c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.ipynb_checkpoints node_modules .cache packages/client/out/* diff --git a/packages/server/src/SettingStore.ts b/packages/server/src/SettingStore.ts index 08546a28..42154af9 100644 --- a/packages/server/src/SettingStore.ts +++ b/packages/server/src/SettingStore.ts @@ -15,7 +15,7 @@ export type SSHConfig = { } export type Connection = { name: string | null, - adapter: 'mysql' | 'postgresql' | 'postgres' | 'sqlite3' | null, + adapter: 'json' | 'mysql' | 'postgresql' | 'postgres' | 'sqlite3' | null, host: string | null port: number | null user: string | null @@ -119,7 +119,13 @@ export default class SettingStore extends EventEmitter.EventEmitter { async setSettingFromWorkspaceConfig(connections: Connection[], projectPath: string = '') { this.personalConfig = { connections } - const extractedPersonalConfig = this.extractPersonalConfigMatchedProjectPath(projectPath) + let extractedPersonalConfig = this.extractPersonalConfigMatchedProjectPath(projectPath) + // Default to first connection if none are matched + if (extractedPersonalConfig == undefined) { + if (connections?.length > 0) { + extractedPersonalConfig = connections[0] + } + } this.setSetting(extractedPersonalConfig || {}) return this.getSetting() } diff --git a/packages/server/src/complete.ts b/packages/server/src/complete.ts index c94add4f..a1978922 100644 --- a/packages/server/src/complete.ts +++ b/packages/server/src/complete.ts @@ -72,9 +72,9 @@ function toCompletionItemFromColumn(column: Column): CompletionItem { } function getTableAndColumnCondidates(tablePrefix: string, schema: Schema, option?: { withoutTable?: boolean, withoutColumn?: boolean }): CompletionItem[] { - const tableCandidates = schema.filter(v => v.tableName.startsWith(tablePrefix)).map(v => toCompletionItemFromTable(v)) + const tableCandidates = schema.tables.filter(v => v.tableName.startsWith(tablePrefix)).map(v => toCompletionItemFromTable(v)) const columnCandidates = Array.prototype.concat.apply([], - schema.filter(v => tableCandidates.map(v => v.label).includes(v.tableName)).map(v => v.columns) + schema.tables.filter(v => tableCandidates.map(v => v.label).includes(v.tableName)).map(v => v.columns) ).map((v: Column) => toCompletionItemFromColumn(v)) const candidates: CompletionItem[] = [] if (!option?.withoutTable) { @@ -120,7 +120,7 @@ function getCandidatedFromIncompleteSubquery(params: { return candidates } -function createTablesFromFromNodes(fromNodes: FromTableNode[]): Schema { +function createTablesFromFromNodes(fromNodes: FromTableNode[]): Table[] { return fromNodes.reduce((p: any, c) => { if (c.type !== 'subquery') { return p @@ -148,7 +148,7 @@ function getCandidatesFromError(target: string, schema: Schema, pos: Pos, e: any return [] } if (candidatesLiterals.includes('.')) { - candidates = candidates.concat(schema.map(v => toCompletionItemFromTable(v))) + candidates = candidates.concat(schema.tables.map(v => toCompletionItemFromTable(v))) } const lastChar = target[target.length - 1] logger.debug(`lastChar: ${lastChar}`) @@ -159,7 +159,7 @@ function getCandidatesFromError(target: string, schema: Schema, pos: Pos, e: any } const tableName = getLastToken(removedLastDotTarget) const subqueryTables = createTablesFromFromNodes(fromNodes) - const attachedAlias = schema.concat(subqueryTables).map(v => { + const attachedAlias = schema.tables.concat(subqueryTables).map(v => { const as = fromNodes.filter((v2: any) => v.tableName === v2.table).map(v => v.as) return Object.assign({}, v, { as: as ? as : [] }) }) @@ -209,7 +209,7 @@ function completeSelectStatement(ast: SelectStatement, _pos: Pos, _schema: Schem return candidates } -export default function complete(sql: string, pos: Pos, schema: Schema = []) { +export default function complete(sql: string, pos: Pos, schema: Schema = {tables:[], functions:[]}) { logger.debug(`complete: ${sql}, ${JSON.stringify(pos)}`) let candidates: CompletionItem[] = [] let error = null; @@ -243,7 +243,7 @@ export default function complete(sql: string, pos: Pos, schema: Schema = []) { if (ast.type === 'select' && Array.isArray(ast.from?.tables)) { const fromTable = getFromNodeByPos(ast.from?.tables || [], pos) if (fromTable && fromTable.type === 'table') { - candidates = candidates.concat(schema.map(v => toCompletionItemFromTable(v))) + candidates = candidates.concat(schema.tables.map(v => toCompletionItemFromTable(v))) .concat([{ label: 'INNER JOIN' }, { label: 'LEFT JOIN' }]) if (fromTable.join && !fromTable.on) { candidates.push({ label: 'ON' }) diff --git a/packages/server/src/createServer.ts b/packages/server/src/createServer.ts index bde65dbc..a65d0704 100644 --- a/packages/server/src/createServer.ts +++ b/packages/server/src/createServer.ts @@ -5,7 +5,7 @@ import { TextDocumentPositionParams, CompletionItem, } from 'vscode-languageserver' -import { TextDocument } from 'vscode-languageserver-textdocument' +import { TextDocument } from 'vscode-languageserver-textdocument' import { CodeAction, TextDocumentEdit, TextEdit, Position, CodeActionKind } from 'vscode-languageserver-types' import cache from './cache' import complete from './complete' @@ -19,10 +19,12 @@ import initializeLogging from './initializeLogging' import { lint, LintResult } from 'sqlint' import log4js from 'log4js' import { RequireSqlite3Error } from './database_libs/Sqlite3Client' +import * as fs from 'fs' +import { RawConfig } from 'sqlint' export type ConnectionMethod = 'node-ipc' | 'stdio' type Args = { - method?: ConnectionMethod + method?: ConnectionMethod } export function createServerWithConnection(connection: Connection) { @@ -30,17 +32,38 @@ export function createServerWithConnection(connection: Connection) { const logger = log4js.getLogger() let documents: TextDocuments = new TextDocuments(TextDocument) documents.listen(connection); - let schema: Schema = [] + let schema: Schema = { tables: [], functions: [] } let hasConfigurationCapability = false let rootPath = '' + let lintConfig: RawConfig | null | undefined - async function makeDiagnostics(document: TextDocument) { - const lintConfig = hasConfigurationCapability && ( - await connection.workspace.getConfiguration({ - section: 'sqlLanguageServer', + // Read schema file + function readJsonSchemaFile(filePath: string) { + logger.info(`loading schema file: ${filePath}`) + const data = fs.readFileSync(filePath, "utf8").replace(/^\ufeff/u, ""); + try { + schema = JSON.parse(data); + } + catch (e) { + logger.error("failed to read schema file") + connection.sendNotification('sqlLanguageServer.error', { + message: "Failed to read schema file: " + filePath + " error: " + e.message }) - )?.lint || {} - const hasRules = lintConfig.hasOwnProperty('rules') + throw e + } + } + + function readAndMonitorJsonSchemaFile(filePath: string) { + readJsonSchemaFile(filePath) + fs.watchFile(filePath, (_curr, _prev) => { + logger.info(`change detected, reloading schema file: ${filePath}`) + readJsonSchemaFile(filePath) + }) + } + + async function makeDiagnostics(document: TextDocument) { + + const hasRules = lintConfig?.hasOwnProperty('rules') const diagnostics = createDiagnostics( document.uri, document.getText(), @@ -56,7 +79,14 @@ export function createServerWithConnection(connection: Connection) { connection.onInitialize((params): InitializeResult => { const capabilities = params.capabilities - hasConfigurationCapability = !!capabilities.workspace && !!capabilities.workspace.configuration; + // JupyterLab sends didChangeConfiguration information + // using both the workspace.configuration and + // workspace.didChangeConfiguration + hasConfigurationCapability = !!capabilities.workspace && ( + !!capabilities.workspace.configuration || + !!capabilities.workspace.didChangeConfiguration); + + logger.debug(`onInitialize: ${params.rootPath}`) rootPath = params.rootPath || '' @@ -79,9 +109,9 @@ export function createServerWithConnection(connection: Connection) { }) connection.onInitialized(async () => { - SettingStore.getInstance().on('change', async () => { + SettingStore.getInstance().on('change', async () => { logger.debug('onInitialize: receive change event from SettingStore') - try { + try { try { connection.sendNotification('sqlLanguageServer.finishSetup', { personalConfig: SettingStore.getInstance().getPersonalConfig(), @@ -90,21 +120,37 @@ export function createServerWithConnection(connection: Connection) { } catch (e) { logger.error(e) } - try { - const client = getDatabaseClient( - SettingStore.getInstance().getSetting() - ) - schema = await client.getSchema() - logger.debug("get schema") - logger.debug(JSON.stringify(schema)) - } catch (e) { - logger.error("failed to get schema info") - if (e instanceof RequireSqlite3Error) { + var setting = SettingStore.getInstance().getSetting() + if (setting.adapter == 'json') { + // Loading schema from json file + var path = setting.filename || "" + if (path == "") { + logger.error("filename must be provided") connection.sendNotification('sqlLanguageServer.error', { - message: "Need to rebuild sqlite3 module." + message: "filename must be provided" }) + throw "filename must be provided" + } + readAndMonitorJsonSchemaFile(path) + } + else { + // Else get schema form database client + try { + const client = getDatabaseClient( + SettingStore.getInstance().getSetting() + ) + schema = await client.getSchema() + logger.debug("get schema") + logger.debug(JSON.stringify(schema)) + } catch (e) { + logger.error("failed to get schema info") + if (e instanceof RequireSqlite3Error) { + connection.sendNotification('sqlLanguageServer.error', { + message: "Need to rebuild sqlite3 module." + }) + } + throw e } - throw e } } catch (e) { logger.error(e) @@ -136,7 +182,9 @@ export function createServerWithConnection(connection: Connection) { SettingStore.getInstance().setSettingFromWorkspaceConfig(connections) } + // On configuration changes we retrieve the lint config const lint = change.settings?.sqlLanguageServer?.lint + lintConfig = lint if (lint?.rules) { documents.all().forEach(v => { makeDiagnostics(v) @@ -149,13 +197,13 @@ export function createServerWithConnection(connection: Connection) { if (!text) { return [] } - logger.debug(text || '') - const candidates = complete(text, { + logger.debug(text || '') + const candidates = complete(text, { line: docParams.position.line, column: docParams.position.character }, schema).candidates logger.debug(candidates.map(v => v.label).join(",")) - return candidates + return candidates }) connection.onCodeAction(params => { @@ -186,9 +234,9 @@ export function createServerWithConnection(connection: Connection) { const edit = v.range.startOffset === v.range.endOffset ? TextEdit.insert(toPosition(text, v.range.startOffset), v.text) : TextEdit.replace({ - start: toPosition(text, v.range.startOffset), - end: toPosition(text, v.range.endOffset) - }, v.text) + start: toPosition(text, v.range.startOffset), + end: toPosition(text, v.range.endOffset) + }, v.text) return edit })) ] @@ -196,7 +244,7 @@ export function createServerWithConnection(connection: Connection) { action.diagnostics = params.context.diagnostics return [action] }) - + connection.onCompletionResolve((item: CompletionItem): CompletionItem => { return item }) diff --git a/packages/server/src/database_libs/AbstractClient.ts b/packages/server/src/database_libs/AbstractClient.ts index 8cc83bc0..200d5618 100644 --- a/packages/server/src/database_libs/AbstractClient.ts +++ b/packages/server/src/database_libs/AbstractClient.ts @@ -21,7 +21,16 @@ export type Table = { tableName: string, columns: Column[] } -export type Schema = Table[] +export type DbFunction = { + name: string, + description: string, +} + +export type Schema = { + tables: Table[], + functions: DbFunction[] +} + export default abstract class AbstractClient { connection: any @@ -36,7 +45,7 @@ export default abstract class AbstractClient { abstract DefaultUser: string async getSchema(): Promise { - let schema: Schema = [] + let schema: Schema = {tables:[], functions: []} const sshConnection = this.settings.ssh?.remoteHost ? new SSHConnection({ endHost: this.settings.ssh.remoteHost, @@ -59,11 +68,11 @@ export default abstract class AbstractClient { } if (!(await this.connect())) { logger.error('AbstractClinet.getSchema: failed to connect database') - return [] + return {tables:[], functions: []} } try { const tables = await this.getTables() - schema = await Promise.all( + schema.tables = await Promise.all( tables.map((v) => this.getColumns(v).then(columns => ({ database: this.settings.database, tableName: v, diff --git a/packages/server/test/complete.test.ts b/packages/server/test/complete.test.ts index bb9ee2bc..da916dce 100644 --- a/packages/server/test/complete.test.ts +++ b/packages/server/test/complete.test.ts @@ -82,7 +82,9 @@ describe('keyword completion', () => { }) }) -const SIMPLE_SCHEMA = [ +const SIMPLE_SCHEMA = { +functions: [], +tables: [ { database: null, tableName: 'TABLE1', @@ -92,6 +94,7 @@ const SIMPLE_SCHEMA = [ ] } ] +} describe('TableName completion', () => { test("complete TableName", () => { @@ -187,7 +190,9 @@ describe('cursor on dot', () => { }) }) -const COMPLEX_SCHEMA = [ +const COMPLEX_SCHEMA = { +functions: [], +tables:[ { database: null, tableName: 'employees', @@ -268,6 +273,7 @@ const COMPLEX_SCHEMA = [ ] }, ] +} test("conplete columns from duplicated alias", () => { const sql = `