From a9e269b52c8578301a1f098f1d175e0148551cc4 Mon Sep 17 00:00:00 2001 From: Matthias Platzer Date: Mon, 3 Jul 2023 17:58:41 +0200 Subject: [PATCH 01/11] Added winston logging - use logger.xxx instead of console.xxx - added express middleware logging (using jsonl) - added LOG_PATH as environment variable - more configs postponed for later iteration --- .gitignore | 1 + docker/.env.example | 1 + packages/server/.env.example | 1 + packages/server/package.json | 3 +- packages/server/src/commands/start.ts | 13 ++-- packages/server/src/index.ts | 15 ++-- packages/server/src/utils/config.ts | 25 +++++++ packages/server/src/utils/index.ts | 5 +- packages/server/src/utils/logger.ts | 100 ++++++++++++++++++++++++++ 9 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 packages/server/src/utils/config.ts create mode 100644 packages/server/src/utils/logger.ts diff --git a/.gitignore b/.gitignore index 9f5ef2e56b5..3ae877768dd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ **/yarn.lock ## logs +logs/**/* **/*.log ## build diff --git a/docker/.env.example b/docker/.env.example index 80fbc3be562..8e66d25eb79 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -4,4 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise +# LOG_PATH=/your_api_key_path/logs # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/.env.example b/packages/server/.env.example index 80fbc3be562..f1fbf99059e 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -4,4 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise +# LOG_PATH=./logs # EXECUTION_MODE=child or main \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index eda69322417..05abd6c938a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -58,7 +58,8 @@ "reflect-metadata": "^0.1.13", "socket.io": "^4.6.1", "sqlite3": "^5.1.6", - "typeorm": "^0.3.6" + "typeorm": "^0.3.6", + "winston": "^3.9.0" }, "devDependencies": { "@types/cors": "^2.8.12", diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index d3efc1eb676..9bd1d64b312 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -3,6 +3,7 @@ import path from 'path' import * as Server from '../index' import * as DataSource from '../DataSource' import dotenv from 'dotenv' +import logger from '../utils/logger' dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true }) @@ -25,11 +26,11 @@ export default class Start extends Command { } async stopProcess() { - console.info('Shutting down Flowise...') + logger.info('Shutting down Flowise...') try { // Shut down the app after timeout if it ever stuck removing pools setTimeout(() => { - console.info('Flowise was forced to shut down after 30 secs') + logger.info('Flowise was forced to shut down after 30 secs') process.exit(processExitCode) }, 30000) @@ -37,7 +38,7 @@ export default class Start extends Command { const serverApp = Server.getInstance() if (serverApp) await serverApp.stopApp() } catch (error) { - console.error('There was an error shutting down Flowise...', error) + logger.error('There was an error shutting down Flowise...', error) } process.exit(processExitCode) } @@ -49,7 +50,7 @@ export default class Start extends Command { // Prevent throw new Error from crashing the app // TODO: Get rid of this and send proper error message to ui process.on('uncaughtException', (err) => { - console.error('uncaughtException: ', err) + logger.error('uncaughtException: ', err) }) const { flags } = await this.parse(Start) @@ -63,11 +64,11 @@ export default class Start extends Command { await (async () => { try { - this.log('Starting Flowise...') + logger.info('Starting Flowise...') await DataSource.init() await Server.start() } catch (error) { - console.error('There was an error starting Flowise...', error) + logger.error('There was an error starting Flowise...', error) processExitCode = EXIT_CODE.FAILED // @ts-ignore process.emit('SIGINT') diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 15762a23c4f..73dbada4f5a 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,6 +6,8 @@ import http from 'http' import * as fs from 'fs' import basicAuth from 'express-basic-auth' import { Server } from 'socket.io' +import logger from './utils/logger' +import { expressRequestLogger } from './utils/logger' import { IChatFlow, @@ -57,13 +59,16 @@ export class App { constructor() { this.app = express() + + // Add the expressRequestLogger middleware to log all requests + this.app.use(expressRequestLogger) } async initDatabase() { // Initialize database this.AppDataSource.initialize() .then(async () => { - console.info('📦[server]: Data Source has been initialized!') + logger.info('📦 [server]: Data Source has been initialized!') // Initialize pools this.nodesPool = new NodesPool() @@ -75,7 +80,7 @@ export class App { await getAPIKeys() }) .catch((err) => { - console.error('❌[server]: Error during Data Source initialization:', err) + logger.error('❌ [server]: Error during Data Source initialization:', err) }) } @@ -614,7 +619,7 @@ export class App { }) }) } catch (err) { - console.error(err) + logger.error(err) } } @@ -792,7 +797,7 @@ export class App { const removePromises: any[] = [] await Promise.all(removePromises) } catch (e) { - console.error(`❌[server]: Flowise Server shut down error: ${e}`) + logger.error(`❌[server]: Flowise Server shut down error: ${e}`) } } } @@ -832,7 +837,7 @@ export async function start(): Promise { await serverApp.config(io) server.listen(port, () => { - console.info(`⚡️[server]: Flowise Server is listening at ${port}`) + logger.info(`⚡️ [server]: Flowise Server is listening at ${port}`) }) } diff --git a/packages/server/src/utils/config.ts b/packages/server/src/utils/config.ts new file mode 100644 index 00000000000..a4a33937952 --- /dev/null +++ b/packages/server/src/utils/config.ts @@ -0,0 +1,25 @@ +// BEWARE: This file is an intereem solution until we have a proper config strategy + +import path from 'path' +import dotenv from 'dotenv' + +dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true }) + +// default config +const loggingConfig = { + dir: process.env.LOG_PATH ?? './logs', + server: { + level: 'info', + filename: 'server.log', + errorFilename: 'server-error.log' + }, + express: { + level: 'info', + format: 'jsonl', // can't be changed currently + filename: 'server-requests.log.jsonl' // should end with .jsonl + } +} + +export default { + logging: loggingConfig +} diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 005f4a4bb47..55529afb0b6 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1,6 +1,7 @@ import path from 'path' import fs from 'fs' import moment from 'moment' +import logger from './logger' import { IComponentNodes, IDepthQueue, @@ -227,7 +228,7 @@ export const buildLangchain = async ( databaseEntities }) } catch (e: any) { - console.error(e) + logger.error(e) throw new Error(e) } @@ -595,7 +596,7 @@ export const replaceAllAPIKeys = async (content: ICommonObject[]): Promise try { await fs.promises.writeFile(getAPIKeyPath(), JSON.stringify(content), 'utf8') } catch (error) { - console.error(error) + logger.error(error) } } diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts new file mode 100644 index 00000000000..277cf3ad5be --- /dev/null +++ b/packages/server/src/utils/logger.ts @@ -0,0 +1,100 @@ +import * as path from 'path' +import * as fs from 'fs' +import config from './config' // should be replaced by node-config or similar +import { createLogger, transports, format } from 'winston' +import { NextFunction, Request, Response } from 'express' + +const { combine, timestamp, printf } = format + +// expect the log dir be relative to the projects root +const logDir = path.join(__dirname, '../../../..', config.logging.dir ?? './logs') + +// Create the log directory if it doesn't exist +if (!fs.existsSync(logDir)) { + fs.mkdirSync(logDir) +} + +const logger = createLogger({ + format: combine( + timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + format.json(), + printf(({ level, message, timestamp }) => { + return `${timestamp} [${level.toUpperCase()}]: ${message}` + }) + ), + defaultMeta: { + package: 'server' + }, + transports: [ + new transports.Console(), + new transports.File({ + filename: path.join(logDir, config.logging.server.filename ?? 'server.log'), + level: config.logging.server.level ?? 'info' + }), + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'), + level: 'error' // Log only errors to this file + }) + ], + exceptionHandlers: [ + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log') + }) + ], + rejectionHandlers: [ + new transports.File({ + filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log') + }) + ] +}) + +/** + * This function is used by express as a middleware. + * @example + * this.app = express() + * this.app.use(expressRequestLogger) + */ +export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void { + const fileLogger = createLogger({ + format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.json()), + defaultMeta: { + package: 'server', + request: { + method: req.method, + url: req.url, + body: req.body, + query: req.query, + params: req.params, + headers: req.headers + } + }, + transports: [ + new transports.File({ + filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'), + level: 'debug' + }) + ] + }) + + const getRequestEmoji = (method: string) => { + const requetsEmojis: Record = { + GET: '⬇️', + POST: '⬆️', + PUT: '🖊', + DELETE: '❌' + } + + return requetsEmojis[method] || '?' + } + + if (req.method !== 'GET') { + fileLogger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + logger.info(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } else { + fileLogger.http(`${getRequestEmoji(req.method)} ${req.method} ${req.url}`) + } + + next() +} + +export default logger From 89511c83950b826bed24964381eb00c1d21bfd7b Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:23:34 +0100 Subject: [PATCH 02/11] Update config.ts --- packages/server/src/utils/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/config.ts b/packages/server/src/utils/config.ts index a4a33937952..d81fe7c3f68 100644 --- a/packages/server/src/utils/config.ts +++ b/packages/server/src/utils/config.ts @@ -7,7 +7,7 @@ dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true } // default config const loggingConfig = { - dir: process.env.LOG_PATH ?? './logs', + dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', 'logs'), server: { level: 'info', filename: 'server.log', From ec777c65eac979172af911fc7498be311fb52af6 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:23:54 +0100 Subject: [PATCH 03/11] Update logger.ts --- packages/server/src/utils/logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 277cf3ad5be..1c28b173cd4 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -7,7 +7,7 @@ import { NextFunction, Request, Response } from 'express' const { combine, timestamp, printf } = format // expect the log dir be relative to the projects root -const logDir = path.join(__dirname, '../../../..', config.logging.dir ?? './logs') +const logDir = config.logging.dir // Create the log directory if it doesn't exist if (!fs.existsSync(logDir)) { From 19758105584ee09bd8e1f431b3dceea249425b4d Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:26:01 +0100 Subject: [PATCH 04/11] Update .env.example --- packages/server/.env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/.env.example b/packages/server/.env.example index f1fbf99059e..262e08a63d6 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -4,5 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise -# LOG_PATH=./logs -# EXECUTION_MODE=child or main \ No newline at end of file +# LOG_PATH=/your_log_path/logs +# EXECUTION_MODE=child or main From 13c4a732eb194267aeba0a7df01f71959bf552bd Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:26:22 +0100 Subject: [PATCH 05/11] Update .env.example --- docker/.env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 8e66d25eb79..262e08a63d6 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -4,5 +4,5 @@ PORT=3000 # DEBUG=true # DATABASE_PATH=/your_database_path/.flowise # APIKEY_PATH=/your_api_key_path/.flowise -# LOG_PATH=/your_api_key_path/logs -# EXECUTION_MODE=child or main \ No newline at end of file +# LOG_PATH=/your_log_path/logs +# EXECUTION_MODE=child or main From 1fc9e9e4362282a3ab8f2a9d667943cb9c82335b Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:27:23 +0100 Subject: [PATCH 06/11] Update docker-compose.yml --- docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 97aea017155..3077c43d372 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,6 +10,7 @@ services: - FLOWISE_PASSWORD=${FLOWISE_PASSWORD} - DATABASE_PATH=${DATABASE_PATH} - APIKEY_PATH=${APIKEY_PATH} + - LOG_PATH=${LOG_PATH} - EXECUTION_MODE=${EXECUTION_MODE} - DEBUG=${DEBUG} ports: From c0f36387a7ca62c44d3dc1a0064cdff76cf1ac5c Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 22:28:17 +0100 Subject: [PATCH 07/11] Update start.ts --- packages/server/src/commands/start.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/commands/start.ts b/packages/server/src/commands/start.ts index 9bd1d64b312..c05c042a6b3 100644 --- a/packages/server/src/commands/start.ts +++ b/packages/server/src/commands/start.ts @@ -22,6 +22,7 @@ export default class Start extends Command { DEBUG: Flags.string(), DATABASE_PATH: Flags.string(), APIKEY_PATH: Flags.string(), + LOG_PATH: Flags.string(), EXECUTION_MODE: Flags.string() } @@ -59,6 +60,7 @@ export default class Start extends Command { if (flags.PORT) process.env.PORT = flags.PORT if (flags.DATABASE_PATH) process.env.DATABASE_PATH = flags.DATABASE_PATH if (flags.APIKEY_PATH) process.env.APIKEY_PATH = flags.APIKEY_PATH + if (flags.LOG_PATH) process.env.LOG_PATH = flags.LOG_PATH if (flags.EXECUTION_MODE) process.env.EXECUTION_MODE = flags.EXECUTION_MODE if (flags.DEBUG) process.env.DEBUG = flags.DEBUG From 7ee2b1194d0290d8904d17ba2e884e796465e94a Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 23:05:51 +0100 Subject: [PATCH 08/11] Only enable logger when DEBUG=true --- packages/server/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 73dbada4f5a..c7206154fcf 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -61,7 +61,7 @@ export class App { this.app = express() // Add the expressRequestLogger middleware to log all requests - this.app.use(expressRequestLogger) + if (process.env.DEBUG === 'true') this.app.use(expressRequestLogger) } async initDatabase() { From da1cfc79c424b1ee82ff515cdb84e5eaccd05de1 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 3 Jul 2023 23:07:31 +0100 Subject: [PATCH 09/11] ignore all files inside logs folder --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3ae877768dd..533f68a5278 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ **/yarn.lock ## logs -logs/**/* +**/logs **/*.log ## build @@ -43,4 +43,4 @@ logs/**/* **/uploads ## compressed -**/*.tgz \ No newline at end of file +**/*.tgz From e977fe7d70db31ecec37abc2265e6545135d82b6 Mon Sep 17 00:00:00 2001 From: Matthias Platzer Date: Thu, 6 Jul 2023 14:29:37 +0200 Subject: [PATCH 10/11] Update config.ts /logs was in /packages -> moved it to project root (one level up) --- packages/server/src/utils/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/config.ts b/packages/server/src/utils/config.ts index d81fe7c3f68..c38d5a0cf0c 100644 --- a/packages/server/src/utils/config.ts +++ b/packages/server/src/utils/config.ts @@ -7,7 +7,7 @@ dotenv.config({ path: path.join(__dirname, '..', '..', '.env'), override: true } // default config const loggingConfig = { - dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', 'logs'), + dir: process.env.LOG_PATH ?? path.join(__dirname, '..', '..', '..', '..', 'logs'), server: { level: 'info', filename: 'server.log', From f83f1c64d3009f0b22cadd63592ac32447eacc2d Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sat, 8 Jul 2023 23:26:28 +0100 Subject: [PATCH 11/11] allow logger by default --- packages/server/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 280d870bd48..38917023886 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -61,7 +61,7 @@ export class App { this.app = express() // Add the expressRequestLogger middleware to log all requests - if (process.env.DEBUG === 'true') this.app.use(expressRequestLogger) + this.app.use(expressRequestLogger) } async initDatabase() {