From 9815a100a6538a2807736e06a1e9ad88e1fbb0fe Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 26 Jan 2024 08:11:56 -0800 Subject: [PATCH] airbyte-ci: override secrets scrubbing in very few cases (#34555) Co-authored-by: Augustin --- airbyte-ci/connectors/pipelines/README.md | 3 ++- .../pipelines/dagger/actions/secrets.py | 18 ++++++++++++++++-- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 4eafa436cd2ce..771e7d03cdcfa 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -578,8 +578,9 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 3.7.2 | [#34555](https://github.com/airbytehq/airbyte/pull/34555) | Override secret masking in some very specific special cases. | | 3.7.1 | [#34441](https://github.com/airbytehq/airbyte/pull/34441) | Support masked secret scrubbing for java CDK v0.15+ | -| 3.7.0 | [#34343](https://github.com/airbytehq/airbyte/pull/34343) | allow running connector upgrade_cdk for java connectors | +| 3.7.0 | [#34343](https://github.com/airbytehq/airbyte/pull/34343) | allow running connector upgrade_cdk for java connectors | | 3.6.1 | [#34490](https://github.com/airbytehq/airbyte/pull/34490) | Fix inconsistent dagger log path typing | | 3.6.0 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Add python registry publishing | | 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index f3c6eac78d5a2..2a943ddd3dd0c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -19,7 +19,18 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext -async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: +# List of overrides for the secrets masking logic. +# These keywords may have been marked as secrets, perhaps somewhat aggressively. +# Masking them, however, is annoying and pointless. +# This list should be extended (carefully) as needed. +NOT_REALLY_SECRETS = { + "admin", + "airbyte", + "host", +} + + +async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container, connector_technical_name: str) -> list[str]: """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. @@ -27,6 +38,9 @@ async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) secrets_to_mask = [] if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): for secret_to_mask in secrets_to_mask_file.splitlines(): + if secret_to_mask in NOT_REALLY_SECRETS or secret_to_mask in connector_technical_name: + # Don't mask secrets which are also common words or connector name. + continue # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked print(f"::add-mask::{secret_to_mask}") @@ -59,7 +73,7 @@ async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = " ) # We don't want to print secrets in the logs when running locally. if context.is_ci: - context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) + context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets, context.connector.technical_name) connector_secrets = {} for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index df0e0928f2763..25012ff9d0ce2 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.7.1" +version = "3.7.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]