Skip to content

Commit

Permalink
airbyte-commons: remove secrets scrubbing from test logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@postamar
postamar committed Jan 23, 2024
1 parent 4b70ac9 commit 3fdf276
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 306 deletions.
8 changes: 0 additions & 8 deletions airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id 'java-library'
id 'de.undercouch.download' version "5.4.0"
}

java {
Expand All @@ -18,10 +17,3 @@ dependencies {
// this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library.
implementation 'com.jayway.jsonpath:json-path:2.7.0'
}

def downloadSpecSecretMask = tasks.register('downloadSpecSecretMask', Download) {
src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml'
dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml')
overwrite true
}
tasks.named('processResources').configure { dependsOn downloadSpecSecretMask }
134 changes: 0 additions & 134 deletions ...e-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java
View file

This file was deleted.

166 changes: 2 additions & 164 deletions airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO" packages="io.airbyte.commons.logging">
<Properties>

<!-- Mask the string apikey=<string> to apikey=***** to prevent secrets leaking. -->
<Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
<Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %m%n</Property>
<!--Logs the timestamp and log_source/application name in the beginning of the line if it exists with a > separator, and then always the rest of the line.-->
<Property name="simple-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>

<Property name="simple-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %m%n</Property>
<!-- Always log INFO by default. -->
<Property name="log-level">${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}}</Property>

<Property name="s3-bucket">${sys:S3_LOG_BUCKET:-${env:S3_LOG_BUCKET}}</Property>
<Property name="s3-region">${sys:S3_LOG_BUCKET_REGION:-${env:S3_LOG_BUCKET_REGION}}</Property>
<Property name="s3-aws-key">${sys:AWS_ACCESS_KEY_ID:-${env:AWS_ACCESS_KEY_ID}}</Property>
<Property name="s3-aws-secret">${sys:AWS_SECRET_ACCESS_KEY:-${env:AWS_SECRET_ACCESS_KEY}}</Property>
<Property name="s3-minio-endpoint">${sys:S3_MINIO_ENDPOINT:-${env:S3_MINIO_ENDPOINT}}</Property>
<Property name="s3-path-style-access">${sys:S3_PATH_STYLE_ACCESS:-${env:S3_PATH_STYLE_ACCESS}}</Property>

<Property name="gcs-log-bucket">${sys:GCS_LOG_BUCKET:-${env:GCS_LOG_BUCKET}}</Property>
</Properties>

<Appenders>
Expand All @@ -34,159 +22,9 @@
</ContextMapFilter>
<PatternLayout pattern="${simple-pattern}"/>
</Console>

<Rewrite name="SecretMaskRewrite">
<MaskedDataInterceptor></MaskedDataInterceptor>
<AppenderRef ref="Default"/>
<AppenderRef ref="SimpleDefault"/>
<AppenderRef ref="LogSplit"/>
<AppenderRef ref="SimpleLogSplit"/>
<AppenderRef ref="LogSplitCloud"/>
<AppenderRef ref="SimpleLogSplitCloud"/>
<AppenderRef ref="AppLogSplit"/>
<AppenderRef ref="AppLogSplitCloud"/>
</Rewrite>

<Routing name="LogSplit">
<ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
<KeyValuePair key="simple" value="true"/>
</ContextMapFilter>
<Routes pattern="$${ctx:job_log_path}">
<!-- Don't split logs if job_root isn't defined -->
<Route key="$${ctx:job_log_path}">
<Null name="/dev/null"/>
</Route>
<Route>
<File name="${ctx:job_log_path}-local" fileName="${ctx:job_log_path}">
<PatternLayout pattern="${default-pattern}"/>
</File>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>

<Routing name="SimpleLogSplit">
<ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
<KeyValuePair key="simple" value="true"/>
</ContextMapFilter>
<Routes pattern="$${ctx:job_log_path}">
<!-- Don't split logs if job_root isn't defined -->
<Route key="$${ctx:job_log_path}">
<Null name="/dev/null"/>
</Route>
<Route>
<File name="${ctx:job_log_path}-local" fileName="${ctx:job_log_path}">
<PatternLayout pattern="${simple-pattern}"/>
</File>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>

<!--
Separate routers are created for each cloud logger as
1) a Route only accepts 1 appender
2) Routes don't support routing log output to more than Route
-->
<Routing name="LogSplitCloud">
<ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
<KeyValuePair key="simple" value="true"/>
</ContextMapFilter>
<Routes pattern="$${ctx:cloud_job_log_path}">
<!-- Don't split logs if job_root isn't defined -->
<Route key="$${ctx:cloud_job_log_path}">
<Null name="/dev/null"/>
</Route>
<Route>
<Log4j2Appender name="${ctx:cloud_job_log_path}"
verbose="true"
stagingBufferAge="1"
s3Bucket="${s3-bucket}" s3Path="job-logging${ctx:cloud_job_log_path}" s3Region="${s3-region}"
s3AwsKey="${s3-aws-key}" s3AwsSecret="${s3-aws-secret}"
s3ServiceEndpoint="${s3-minio-endpoint}" s3PathStyleAccess="${s3-path-style-access}"
gcpStorageBucket="${gcs-log-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:cloud_job_log_path}">
<PatternLayout pattern="${default-pattern}"/>
</Log4j2Appender>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>

<Routing name="SimpleLogSplitCloud">
<ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
<KeyValuePair key="simple" value="true"/>
</ContextMapFilter>
<Routes pattern="$${ctx:cloud_job_log_path}">
<!-- Don't split logs if job_root isn't defined -->
<Route key="$${ctx:cloud_job_log_path}">
<Null name="/dev/null"/>
</Route>
<Route>
<Log4j2Appender name="${ctx:cloud_job_log_path}"
verbose="true"
stagingBufferAge="1"
s3Bucket="${s3-bucket}" s3Path="job-logging${ctx:cloud_job_log_path}" s3Region="${s3-region}"
s3AwsKey="${s3-aws-key}" s3AwsSecret="${s3-aws-secret}"
s3ServiceEndpoint="${s3-minio-endpoint}" s3PathStyleAccess="${s3-path-style-access}"
gcpStorageBucket="${gcs-log-bucket}" gcpStorageBlobNamePrefix="job-logging${ctx:cloud_job_log_path}">
<PatternLayout pattern="${simple-pattern}"/>
</Log4j2Appender>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>

<Routing name="AppLogSplit">
<Routes pattern="$${ctx:workspace_app_root}">
<!-- Don't split logs if workspace_app_log_root isn't defined -->
<Route key="$${ctx:workspace_app_root}">
<Null name="/dev/null"/>
</Route>
<Route>
<RollingFile
name="${ctx:workspace_app_root}-local"
fileName="${ctx:workspace_app_root}/logs.log"
filePattern="${ctx:workspace_app_root}/logs.%i.log.gz"
ignoreExceptions="false">
<PatternLayout pattern="${default-pattern}"/>
<Policies>
<SizeBasedTriggeringPolicy size="100MB" />
</Policies>
<DefaultRolloverStrategy max="3" />
</RollingFile>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>
<Routing name="AppLogSplitCloud">
<Routes pattern="$${ctx:cloud_workspace_app_root}">
<!-- Don't split logs if workspace_app_log_root isn't defined -->
<Route key="$${ctx:cloud_workspace_app_root}">
<Null name="/dev/null"/>
</Route>
<Route>
<Log4j2Appender name="app-logging/${ctx:cloud_workspace_app_root}/"
stagingBufferAge="1"
s3Bucket="${s3-bucket}" s3Path="app-logging${ctx:cloud_workspace_app_root}" s3Region="${s3-region}"
s3AwsKey="${s3-aws-key}" s3AwsSecret="${s3-aws-secret}"
s3ServiceEndpoint="${s3-minio-endpoint}" s3PathStyleAccess="${s3-path-style-access}"
gcpStorageBucket="${gcs-log-bucket}" gcpStorageBlobNamePrefix="app-logging${ctx:cloud_workspace_app_root}">
<PatternLayout pattern="${default-pattern}"/>
</Log4j2Appender>
</Route>
</Routes>
<IdlePurgePolicy timeToLive="15" timeUnit="minutes"/>
</Routing>
</Appenders>

<Loggers>
<Root level="${log-level}">
<!-- Use the rewrite policy to ensure that connector configuration secrets are masked when logged
Any additional appender references should be added to the <Rewrite> section above in this document
to ensure that any secrets logged via those appenders is masked.-->
<AppenderRef ref="SecretMaskRewrite"/>
</Root>

<Logger name="org.eclipse.jetty" level="INFO" />
<Logger name="com.github.dockerjava" level="INFO" />
<Logger name="org.apache.hc" level="INFO" />
Expand Down

0 comments on commit 3fdf276

Please sign in to comment.