SHARD-2741 - feat: log file rotation strategy changes

src/config/logs.ts

@@ -7,20 +7,137 @@ const LOGS_CONFIG: StrictLogsConfiguration = {
   options: {
     appenders: {
       out: { type: 'console', maxLogSize: 10000000, backups: 10 },
-      seq: { type: 'file', maxLogSize: 1000000000, backups: 10 },
-      main: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      app: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      p2p: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      snapshot: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      cycle: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      fatal: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      exit: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      errorFile: { type: 'file', maxLogSize: 10000000, backups: 10 },
+      seq: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      main: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      app: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      p2p: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      snapshot: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      cycle: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      fatal: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      exit: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      errorFile: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
       errors: { type: 'logLevelFilter', level: 'ERROR', appender: 'errorFile' },
-      net: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      playback: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      shardDump: { type: 'file', maxLogSize: 10000000, backups: 10 },
-      statsDump: { type: 'file', maxLogSize: 10000000, backups: 10 },
+      net: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      playback: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      shardDump: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
+      statsDump: {
+        type: 'dateFileWithSize',
+        pattern: 'yyyy-MM-dd-HH-mm-ss',
+        keepFileExt: true,
+        maxLogSize: 10485760,
+        backups: 10,
+        numBackups: 10,
+        compress: false,
+        alwaysIncludePattern: false,
+      },
     },
     categories: {
       default: { appenders: ['out'], level: 'trace' },

src/logger/appenders/customSizeTime.ts

@@ -0,0 +1,100 @@
+import path from 'path'
+import { RollingFileStream } from 'streamroller'
+
+type Layouts = {
+  patternLayout?: (pattern: string) => (evt: any) => string
+}
+
+type AppenderConfig = {
+  filename?: string
+  maxLogSize?: number
+  backups?: number
+  pattern?: string
+  layout?: string
+}
+
+let stream: RollingFileStream | null = null
+let bytesWritten = 0
+let lastStamp = ''
+let sameStampIndex = 0
+
+function format(date: Date, pattern: string): string {
+  const yyyy = String(date.getFullYear())
+  const MM = String(date.getMonth() + 1).padStart(2, '0')
+  const dd = String(date.getDate()).padStart(2, '0')
+  const HH = String(date.getHours()).padStart(2, '0')
+  const mm = String(date.getMinutes()).padStart(2, '0')
+  const ss = String(date.getSeconds()).padStart(2, '0')
+  const SSS = String(date.getMilliseconds()).padStart(3, '0')
+  return pattern
+    .replace('yyyy', yyyy)
+    .replace('MM', MM)
+    .replace('dd', dd)
+    .replace('HH', HH)
+    .replace('mm', mm)
+    .replace('ss', ss)
+    .replace('SSS', SSS)
+}
+
+function buildFilePath(baseFile: string, pattern: string): string {
+  const dir = path.dirname(baseFile)
+  const base = path.basename(baseFile, path.extname(baseFile))
+  const ext = path.extname(baseFile) || '.log'
+  const stamp = format(new Date(), pattern)
+  if (stamp === lastStamp) {
+    sameStampIndex += 1
+  } else {
+    sameStampIndex = 0
+    lastStamp = stamp
+  }
+  const suffix = sameStampIndex > 0 ? `.${String(sameStampIndex).padStart(2, '0')}` : ''
+  return path.join(dir, `${base}.${stamp}${suffix}${ext}`)
+}
+
+export function configure(config: AppenderConfig = {}, layouts: Layouts) {
+  const maxSize = config.maxLogSize ?? 10_000_000
+  const backups = config.backups ?? 10
+  const pattern = config.pattern ?? 'yyyy-MM-dd-HH-mm-ss'
+  const baseFile = config.filename ?? 'main.log'
+
+  const layoutFn =
+    layouts && layouts.patternLayout && config.layout
+      ? layouts.patternLayout(config.layout)
+      : (evt: any) =>
+          `${evt.startTime.toISOString()} [${evt.level.levelStr}] ${evt.categoryName} - ${evt.data.join(' ')}\n`
+
+  function openNewStream() {
+    const filePath = buildFilePath(baseFile, pattern)
+    stream = new RollingFileStream(filePath, maxSize, backups)
+    bytesWritten = 0
+  }
+
+  return (loggingEvent: any) => {
+    if (!stream) openNewStream()
+    const line = layoutFn(loggingEvent)
+    const sz = Buffer.byteLength(line)
+    if (bytesWritten + sz > maxSize) {
+      try {
+        stream?.end()
+      } catch {}
+      stream = null
+      openNewStream()
+    }
+    stream!.write(line)
+    bytesWritten += sz
+  }
+}
+
+export function shutdown(done: (err?: Error) => void) {
+  try {
+    if (stream) {
+      const s = stream
+      stream = null
+      s.end(() => done())
+      return
+    }
+  } catch (e) {
+    // ignore
+  }
+  done()
+}

src/logger/appenders/dateFileWithSize.ts

@@ -0,0 +1,103 @@
+import path from 'path'
+import { RollingFileStream } from 'streamroller'
+
+type Layouts = {
+  patternLayout?: (pattern: string) => (evt: any) => string
+}
+
+type AppenderConfig = {
+  filename?: string
+  maxLogSize?: number
+  backups?: number
+  pattern?: string
+  layout?: string
+}
+
+function format(date: Date, pattern: string): string {
+  const yyyy = String(date.getFullYear())
+  const MM = String(date.getMonth() + 1).padStart(2, '0')
+  const dd = String(date.getDate()).padStart(2, '0')
+  const HH = String(date.getHours()).padStart(2, '0')
+  const mm = String(date.getMinutes()).padStart(2, '0')
+  const ss = String(date.getSeconds()).padStart(2, '0')
+  const SSS = String(date.getMilliseconds()).padStart(3, '0')
+  return pattern
+    .replace('yyyy', yyyy)
+    .replace('MM', MM)
+    .replace('dd', dd)
+    .replace('HH', HH)
+    .replace('mm', mm)
+    .replace('ss', ss)
+    .replace('SSS', SSS)
+}
+
+export function configure(config: AppenderConfig = {}, layouts: Layouts) {
+  const maxSize = config.maxLogSize ?? 10_000_000
+  const backups = config.backups ?? 10
+  const pattern = config.pattern ?? 'yyyy-MM-dd-HH-mm-ss'
+  const baseFile = config.filename ?? 'main.log'
+
+  // Per-instance state
+  let stream: RollingFileStream | null = null
+  let bytesWritten = 0
+  let lastStamp = ''
+  let sameStampIndex = 0
+
+  function buildFilePath(localBaseFile: string, localPattern: string): string {
+    const dir = path.dirname(localBaseFile)
+    const base = path.basename(localBaseFile, path.extname(localBaseFile))
+    const ext = path.extname(localBaseFile) || '.log'
+    const stamp = format(new Date(), localPattern)
+    if (stamp === lastStamp) {
+      sameStampIndex += 1
+    } else {
+      sameStampIndex = 0
+      lastStamp = stamp
+    }
+    const suffix = sameStampIndex > 0 ? `.${String(sameStampIndex).padStart(2, '0')}` : ''
+    return path.join(dir, `${base}.${stamp}${suffix}${ext}`)
+  }
+
+  const layoutFn =
+    layouts && layouts.patternLayout && config.layout
+      ? layouts.patternLayout(config.layout)
+      : (evt: any) =>
+          `${evt.startTime.toISOString()} [${evt.level.levelStr}] ${evt.categoryName} - ${evt.data.join(' ')}\n`
+
+  function openNewStream() {
+    const filePath = buildFilePath(baseFile, pattern)
+    stream = new RollingFileStream(filePath, maxSize, backups)
+    bytesWritten = 0
+  }
+
+  const appender = (loggingEvent: any) => {
+    if (!stream) openNewStream()
+    const line = layoutFn(loggingEvent)
+    const sz = Buffer.byteLength(line)
+    if (bytesWritten + sz > maxSize) {
+      try {
+        stream?.end()
+      } catch {}
+      stream = null
+      openNewStream()
+    }
+    stream!.write(line)
+    bytesWritten += sz
+  }
+
+  ;(appender as any).shutdown = (done: (err?: Error) => void) => {
+    try {
+      if (stream) {
+        const s = stream
+        stream = null
+        s.end(() => done())
+        return
+      }
+    } catch (e) {
+      // ignore
+    }
+    done()
+  }
+
+  return appender
+}