From f0018943b3217b7cf96fdd2c138150dfe231e67e Mon Sep 17 00:00:00 2001 From: billsedison Date: Sat, 23 Aug 2025 21:40:47 +0800 Subject: [PATCH] #8 Winston Logger ECS Deployment Fix --- .env.example | 3 + README.md | 38 ++++++++ src/common/constants/config.constants.ts | 2 + src/common/services/logger.service.ts | 105 +++++++++++++++-------- src/config/sections/app.config.ts | 1 + src/config/validation.ts | 1 + 6 files changed, 115 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index ca7bca4..63f0eec 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,9 @@ LOG_LEVEL=info # The directory to store log files LOG_DIR=logs +# leave unset (defaults to false in production) +ENABLE_FILE_LOGGING=true + # ------------------------------------- # Kafka Configuration # ------------------------------------- diff --git a/README.md b/README.md index 95aa2df..753447a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,9 @@ Open the `.env` file and configure the variables for your environment. It is cru PORT=3000 LOG_LEVEL=debug LOG_DIR=logs + # Enable file logging in production environments (default: false) + # In development environments, file logging is enabled by default + ENABLE_FILE_LOGGING=false # ------------------------------------- # Kafka Configuration @@ -206,6 +209,31 @@ tsconfig.json # TypeScript config README.md # Documentation ``` +## Logging Configuration + +The autopilot service uses a flexible logging strategy that adapts to different deployment environments: + +### Development Environment +- **File Logging**: Enabled by default for debugging purposes +- **Console Logging**: Always enabled with colorized output +- **Log Files**: Created in the `logs/` directory (configurable via `LOG_DIR`) + +### Production Environment (ECS) +- **File Logging**: Disabled by default to support read-only filesystems +- **Console Logging**: Primary logging method, suitable for container log collection +- **Override**: Set `ENABLE_FILE_LOGGING=true` to enable file logging if the filesystem is writable + +### Environment Variables +- `LOG_LEVEL`: Controls log verbosity (`error`, `warn`, `info`, `debug`, `verbose`) +- `LOG_DIR`: Directory for log files (default: `logs`) +- `ENABLE_FILE_LOGGING`: Boolean flag to control file logging in production + +### ECS Deployment Notes +When deploying to ECS, ensure: +1. `NODE_ENV=production` is set +2. `ENABLE_FILE_LOGGING` is either unset or set to `false` for read-only filesystems +3. Container logging is configured to collect stdout/stderr logs + ## Further Documentation For more detailed technical information, please see the documents in the `docs/` directory: @@ -228,3 +256,13 @@ For more detailed technical information, please see the documents in the `docs/` ``` - **API Authentication Errors**: Ensure Auth0 credentials (`AUTH0_URL`, `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_AUDIENCE`) in your `.env` file are valid and not expired. + +- **ECS Deployment - Read-only Filesystem Error**: If the application fails to start in ECS with errors about creating a 'logs' directory: + - Ensure `NODE_ENV=production` is set in your ECS task definition + - Verify `ENABLE_FILE_LOGGING` is not set or is set to `false` + - Check that your ECS task definition is configured to collect logs from stdout/stderr + +- **Logging Issues**: + - **No log files in development**: Check that `ENABLE_FILE_LOGGING` is not set to `false` + - **File permission errors**: Ensure the application has write permissions to the `LOG_DIR` directory + - **Missing logs in production**: Verify your container logging configuration is collecting stdout/stderr output diff --git a/src/common/constants/config.constants.ts b/src/common/constants/config.constants.ts index 58ab51e..1453868 100644 --- a/src/common/constants/config.constants.ts +++ b/src/common/constants/config.constants.ts @@ -3,6 +3,7 @@ export interface AppConfig { DEFAULT_NODE_ENV: 'development' | 'production' | 'test'; DEFAULT_LOG_LEVEL: 'error' | 'warn' | 'info' | 'debug' | 'verbose'; DEFAULT_LOG_DIR: string; + DEFAULT_ENABLE_FILE_LOGGING: boolean; } export interface KafkaConfig { @@ -51,6 +52,7 @@ export const CONFIG: Config = { DEFAULT_NODE_ENV: 'development', DEFAULT_LOG_LEVEL: 'info', DEFAULT_LOG_DIR: 'logs', + DEFAULT_ENABLE_FILE_LOGGING: false, }, KAFKA: { DEFAULT_CLIENT_ID: 'autopilot-service', diff --git a/src/common/services/logger.service.ts b/src/common/services/logger.service.ts index 85a2f98..5341f09 100644 --- a/src/common/services/logger.service.ts +++ b/src/common/services/logger.service.ts @@ -17,6 +17,58 @@ export class LoggerService implements NestLoggerService { } private createWinstonLogger(): Logger { + const baseTransports: any[] = [ + new transports.Console({ + level: process.env.LOG_LEVEL || CONFIG.APP.DEFAULT_LOG_LEVEL, + format: format.combine( + format.colorize(), + format.printf(({ timestamp, level, message, context, ...meta }) => { + const formattedTimestamp = + timestamp instanceof Date + ? timestamp.toISOString() + : String(timestamp); + const formattedContext = + typeof context === 'string' ? context : 'App'; + const formattedMessage = + typeof message === 'string' + ? message + : typeof message === 'object' && message !== null + ? JSON.stringify(message) + : String(message); + + return `${formattedTimestamp} [${formattedContext}] ${String(level)}: ${formattedMessage}${ + Object.keys(meta).length + ? ' ' + JSON.stringify(meta, null, 1) + : '' + }`; + }), + ), + }), + ]; + + // Add file transports conditionally based on environment + const shouldEnableFileLogging = this.shouldEnableFileLogging(); + if (shouldEnableFileLogging) { + try { + baseTransports.push( + new transports.File({ + filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/error.log`, + level: 'error', + }), + new transports.File({ + filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/combined.log`, + }), + ); + } catch (error) { + // If file transport creation fails (e.g., read-only filesystem), + // log a warning but continue with console-only logging + console.warn( + 'Failed to initialize file transports, falling back to console-only logging:', + error instanceof Error ? error.message : String(error), + ); + } + } + return createLogger({ format: format.combine( format.timestamp(), @@ -24,44 +76,27 @@ export class LoggerService implements NestLoggerService { format.json(), ), defaultMeta: { context: this.context }, - transports: [ - new transports.Console({ - level: process.env.LOG_LEVEL || CONFIG.APP.DEFAULT_LOG_LEVEL, - format: format.combine( - format.colorize(), - format.printf(({ timestamp, level, message, context, ...meta }) => { - const formattedTimestamp = - timestamp instanceof Date - ? timestamp.toISOString() - : String(timestamp); - const formattedContext = - typeof context === 'string' ? context : 'App'; - const formattedMessage = - typeof message === 'string' - ? message - : typeof message === 'object' && message !== null - ? JSON.stringify(message) - : String(message); - - return `${formattedTimestamp} [${formattedContext}] ${String(level)}: ${formattedMessage}${ - Object.keys(meta).length - ? ' ' + JSON.stringify(meta, null, 1) - : '' - }`; - }), - ), - }), - new transports.File({ - filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/error.log`, - level: 'error', - }), - new transports.File({ - filename: `${process.env.LOG_DIR || CONFIG.APP.DEFAULT_LOG_DIR}/combined.log`, - }), - ], + transports: baseTransports, }); } + /** + * Determines whether file logging should be enabled based on environment configuration + */ + private shouldEnableFileLogging(): boolean { + const nodeEnv = process.env.NODE_ENV; + const enableFileLogging = process.env.ENABLE_FILE_LOGGING; + + // In production, only enable file logging if explicitly requested + if (nodeEnv === 'production') { + return enableFileLogging === 'true'; + } + + // In non-production environments (development, test), enable file logging by default + // unless explicitly disabled + return enableFileLogging !== 'false'; + } + private formatMessage(message: unknown): string { if (typeof message === 'string') { return message; diff --git a/src/config/sections/app.config.ts b/src/config/sections/app.config.ts index 3c54a59..3d0b75e 100644 --- a/src/config/sections/app.config.ts +++ b/src/config/sections/app.config.ts @@ -6,5 +6,6 @@ export default registerAs('app', () => ({ logging: { level: process.env.LOG_LEVEL || 'info', directory: process.env.LOG_DIR || 'logs', + enableFileLogging: process.env.ENABLE_FILE_LOGGING === 'true', }, })); diff --git a/src/config/validation.ts b/src/config/validation.ts index 2c41925..291ad2b 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -11,6 +11,7 @@ export const validationSchema = Joi.object({ .valid('error', 'warn', 'info', 'debug', 'verbose') .default('info'), LOG_DIR: Joi.string().default('logs'), + ENABLE_FILE_LOGGING: Joi.boolean().default(false), // Kafka Configuration KAFKA_BROKERS: Joi.string().required(),