HDDS-7082. Delete out of date audit logs (#3647)

apache · Feb 6, 2023 · 471f1b2 · 471f1b2
1 parent db1ea10
commit 471f1b2
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 16 deletions.
diff --git a/hadoop-hdds/docs/content/tools/LogsInOzone.md b/hadoop-hdds/docs/content/tools/LogsInOzone.md
@@ -0,0 +1,45 @@
+---
+title: "Logs in Ozone"
+date: 2023-01-30
+summary: Logs in Ozone.
+---
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# AuditLog
+
+AuditLogs configurations are set in "*-audit-log4j2.properties" files. We
+can change the corresponding files to update the audit log policies for 
+each component.
+
+## Deletion of AuditLog
+
+The default log appender is Rolling appender, the following configurations
+can be added for deletion of out-of-date AuditLogs.
+
+```
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.delete.type=Delete
+appender.rolling.strategy.delete.basePath=${sys:hadoop.log.dir}
+appender.rolling.strategy.delete.maxDepth=1
+appender.rolling.strategy.delete.ifFileName.type=IfFileName
+appender.rolling.strategy.delete.ifFileName.glob=om-audit-*.log.gz
+appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
+appender.rolling.strategy.delete.ifLastModified.age=30d
+```
+
+For more details, please check [Log4j2 Delete on Rollover](https://logging.apache.org/log4j/2.x/manual/appenders.html#CustomDeleteOnRollover).
diff --git a/hadoop-ozone/dist/src/shell/conf/dn-audit-log4j2.properties b/hadoop-ozone/dist/src/shell/conf/dn-audit-log4j2.properties
@@ -62,10 +62,12 @@ filter.write.onMismatch=NEUTRAL
 # Comment this line when using both console and rolling appenders
 appenders=rolling
 
-#Rolling File Appender with size & time thresholds.
-#Rolling is triggered when either threshold is breached.
-#The rolled over file is compressed by default
-#Time interval is specified in seconds 86400s=1 day
+# Rolling File Appender with size & time thresholds.
+# Rolling is triggered when either threshold is breached.
+# The rolled over file is compressed by default
+# Time interval is specified in seconds 86400s=1 day
+# Audit files under the base directory that are 30 days old
+# or older are deleted at rollover time
 appender.rolling.type=RollingFile
 appender.rolling.name=RollingFile
 appender.rolling.fileName =${sys:hadoop.log.dir}/dn-audit-${hostName}.log
@@ -77,6 +79,14 @@ appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval=86400
 appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
 appender.rolling.policies.size.size=64MB
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.delete.type=Delete
+appender.rolling.strategy.delete.basePath=${sys:hadoop.log.dir}
+appender.rolling.strategy.delete.maxDepth=1
+appender.rolling.strategy.delete.ifFileName.type=IfFileName
+appender.rolling.strategy.delete.ifFileName.glob=dn-audit-*.log.gz
+appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
+appender.rolling.strategy.delete.ifLastModified.age=30d
 
 loggers=audit
 logger.audit.type=AsyncLogger

diff --git a/hadoop-ozone/dist/src/shell/conf/om-audit-log4j2.properties b/hadoop-ozone/dist/src/shell/conf/om-audit-log4j2.properties
@@ -62,10 +62,12 @@ filter.write.onMismatch=NEUTRAL
 # Comment this line when using both console and rolling appenders
 appenders=rolling
 
-#Rolling File Appender with size & time thresholds.
-#Rolling is triggered when either threshold is breached.
-#The rolled over file is compressed by default
-#Time interval is specified in seconds 86400s=1 day
+# Rolling File Appender with size & time thresholds.
+# Rolling is triggered when either threshold is breached.
+# The rolled over file is compressed by default
+# Time interval is specified in seconds 86400s=1 day
+# Audit files under the base directory that are 30 days old
+# or older are deleted at rollover time
 appender.rolling.type=RollingFile
 appender.rolling.name=RollingFile
 appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
@@ -77,6 +79,14 @@ appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval=86400
 appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
 appender.rolling.policies.size.size=64MB
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.delete.type=Delete
+appender.rolling.strategy.delete.basePath=${sys:hadoop.log.dir}
+appender.rolling.strategy.delete.maxDepth=1
+appender.rolling.strategy.delete.ifFileName.type=IfFileName
+appender.rolling.strategy.delete.ifFileName.glob=om-audit-*.log.gz
+appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
+appender.rolling.strategy.delete.ifLastModified.age=30d
 
 loggers=audit
 logger.audit.type=AsyncLogger

diff --git a/hadoop-ozone/dist/src/shell/conf/s3g-audit-log4j2.properties b/hadoop-ozone/dist/src/shell/conf/s3g-audit-log4j2.properties
@@ -62,10 +62,12 @@ filter.write.onMismatch=NEUTRAL
 # Comment this line when using both console and rolling appenders
 appenders=rolling
 
-#Rolling File Appender with size & time thresholds.
-#Rolling is triggered when either threshold is breached.
-#The rolled over file is compressed by default
-#Time interval is specified in seconds 86400s=1 day
+# Rolling File Appender with size & time thresholds.
+# Rolling is triggered when either threshold is breached.
+# The rolled over file is compressed by default
+# Time interval is specified in seconds 86400s=1 day
+# Audit files under the base directory that are 30 days old
+# or older are deleted at rollover time
 appender.rolling.type=RollingFile
 appender.rolling.name=RollingFile
 appender.rolling.fileName =${sys:hadoop.log.dir}/s3g-audit-${hostName}.log
@@ -77,6 +79,14 @@ appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval=86400
 appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
 appender.rolling.policies.size.size=64MB
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.delete.type=Delete
+appender.rolling.strategy.delete.basePath=${sys:hadoop.log.dir}
+appender.rolling.strategy.delete.maxDepth=1
+appender.rolling.strategy.delete.ifFileName.type=IfFileName
+appender.rolling.strategy.delete.ifFileName.glob=s3g-audit-*.log.gz
+appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
+appender.rolling.strategy.delete.ifLastModified.age=30d
 
 loggers=audit
 logger.audit.type=AsyncLogger

diff --git a/hadoop-ozone/dist/src/shell/conf/scm-audit-log4j2.properties b/hadoop-ozone/dist/src/shell/conf/scm-audit-log4j2.properties
@@ -62,10 +62,12 @@ filter.write.onMismatch=NEUTRAL
 # Comment this line when using both console and rolling appenders
 appenders=rolling
 
-#Rolling File Appender with size & time thresholds.
-#Rolling is triggered when either threshold is breached.
-#The rolled over file is compressed by default
-#Time interval is specified in seconds 86400s=1 day
+# Rolling File Appender with size & time thresholds.
+# Rolling is triggered when either threshold is breached.
+# The rolled over file is compressed by default
+# Time interval is specified in seconds 86400s=1 day
+# Audit files under the base directory that are 30 days old
+# or older are deleted at rollover time
 appender.rolling.type=RollingFile
 appender.rolling.name=RollingFile
 appender.rolling.fileName =${sys:hadoop.log.dir}/scm-audit-${hostName}.log
@@ -77,6 +79,14 @@ appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval=86400
 appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
 appender.rolling.policies.size.size=64MB
+appender.rolling.strategy.type=DefaultRolloverStrategy
+appender.rolling.strategy.delete.type=Delete
+appender.rolling.strategy.delete.basePath=${sys:hadoop.log.dir}
+appender.rolling.strategy.delete.maxDepth=1
+appender.rolling.strategy.delete.ifFileName.type=IfFileName
+appender.rolling.strategy.delete.ifFileName.glob=scm-audit-*.log.gz
+appender.rolling.strategy.delete.ifLastModified.type=IfLastModified
+appender.rolling.strategy.delete.ifLastModified.age=30d
 
 loggers=audit
 logger.audit.type=AsyncLogger