diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle index 9073e823a0fe25..c92e3bdac1ecbe 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'de.undercouch.download' version "5.4.0" } java { @@ -18,10 +17,3 @@ dependencies { // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. implementation 'com.jayway.jsonpath:json-path:2.7.0' } - -def downloadSpecSecretMask = tasks.register('downloadSpecSecretMask', Download) { - src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' - dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') - overwrite true -} -tasks.named('processResources').configure { dependsOn downloadSpecSecretMask } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java deleted file mode 100644 index 072884392adc1f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.logging; - -import com.fasterxml.jackson.core.type.TypeReference; -import io.airbyte.commons.constants.AirbyteSecretConstants; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import java.nio.charset.Charset; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.impl.Log4jLogEvent; -import org.apache.logging.log4j.message.SimpleMessage; -import org.apache.logging.log4j.status.StatusLogger; - -/** - * Custom Log4j2 {@link RewritePolicy} used to intercept all log messages and mask any JSON - * properties in the message that match the list of maskable properties. - *

- * The maskable properties file is generated by a Gradle task in the {@code :airbyte-config:specs} - * project. The file is named {@code specs_secrets_mask.yaml} and is located in the - * {@code src/main/resources/seed} directory of the {@link :airbyte-config:init} project. - */ -@Plugin(name = "MaskedDataInterceptor", - category = "Core", - elementType = "rewritePolicy", - printObject = true) -public class MaskedDataInterceptor implements RewritePolicy { - - protected static final Logger logger = StatusLogger.getLogger(); - - /** - * The pattern used to determine if a message contains sensitive data. - */ - private final Optional pattern; - - @PluginFactory - public static MaskedDataInterceptor createPolicy( - @PluginAttribute(value = "specMaskFile", - defaultString = "/seed/specs_secrets_mask.yaml") final String specMaskFile) { - return new MaskedDataInterceptor(specMaskFile); - } - - private MaskedDataInterceptor(final String specMaskFile) { - this.pattern = buildPattern(specMaskFile); - } - - @Override - public LogEvent rewrite(final LogEvent source) { - return Log4jLogEvent.newBuilder() - .setLoggerName(source.getLoggerName()) - .setMarker(source.getMarker()) - .setLoggerFqcn(source.getLoggerFqcn()) - .setLevel(source.getLevel()) - .setMessage(new SimpleMessage(applyMask(source.getMessage().getFormattedMessage()))) - .setThrown(source.getThrown()) - .setContextMap(source.getContextMap()) - .setContextStack(source.getContextStack()) - .setThreadName(source.getThreadName()) - .setSource(source.getSource()) - .setTimeMillis(source.getTimeMillis()) - .build(); - } - - /** - * Applies the mask to the message, if necessary. - * - * @param message The log message. - * @return The possibly masked log message. - */ - private String applyMask(final String message) { - if (pattern.isPresent()) { - return message.replaceAll(pattern.get(), "\"$1\":\"" + AirbyteSecretConstants.SECRETS_MASK + "\""); - } else { - return message; - } - } - - /** - * Loads the maskable properties from the provided file. - * - * @param specMaskFile The spec mask file. - * @return The set of maskable properties. - */ - private Set getMaskableProperties(final String specMaskFile) { - - try { - final String maskFileContents = IOUtils.toString(getClass().getResourceAsStream(specMaskFile), Charset.defaultCharset()); - final Map> properties = Jsons.object(Yamls.deserialize(maskFileContents), new TypeReference<>() {}); - return properties.getOrDefault("properties", Set.of()); - } catch (final Exception e) { - logger.error("Unable to load mask data from '{}': {}.", specMaskFile, e.getMessage()); - return Set.of(); - } - } - - /** - * Builds the maskable property matching pattern. - * - * @param specMaskFile The spec mask file. - * @return The regular expression pattern used to find maskable properties. - */ - private Optional buildPattern(final String specMaskFile) { - final Set maskableProperties = getMaskableProperties(specMaskFile); - return !maskableProperties.isEmpty() ? Optional.of(generatePattern(maskableProperties)) : Optional.empty(); - } - - /** - * Generates the property matching pattern string from the provided set of properties. - * - * @param properties The set of properties to match. - * @return The generated regular expression pattern used to match the maskable properties. - */ - private String generatePattern(final Set properties) { - final StringBuilder builder = new StringBuilder(); - builder.append("(?i)"); // case insensitive - builder.append("\"("); - builder.append(properties.stream().collect(Collectors.joining("|"))); - builder.append(")\"\\s*:\\s*(\"(?:[^\"\\\\]|\\\\.)*\"|\\[[^]\\[]*]|\\d+)"); - return builder.toString(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml index 22d52667d0ee0c..3f5caf0947019d 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml @@ -1,23 +1,11 @@ - - - %d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n + %d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %m%n - %d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n - + %d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %m%n ${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}} - - ${sys:S3_LOG_BUCKET:-${env:S3_LOG_BUCKET}} - ${sys:S3_LOG_BUCKET_REGION:-${env:S3_LOG_BUCKET_REGION}} - ${sys:AWS_ACCESS_KEY_ID:-${env:AWS_ACCESS_KEY_ID}} - ${sys:AWS_SECRET_ACCESS_KEY:-${env:AWS_SECRET_ACCESS_KEY}} - ${sys:S3_MINIO_ENDPOINT:-${env:S3_MINIO_ENDPOINT}} - ${sys:S3_PATH_STYLE_ACCESS:-${env:S3_PATH_STYLE_ACCESS}} - - ${sys:GCS_LOG_BUCKET:-${env:GCS_LOG_BUCKET}} @@ -34,159 +22,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -