connectors-ci: mask secrets in GHA logs with ::add-mask:: (#27087)

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py

@@ -181,8 +181,8 @@ async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> C
     """
     python_base_environment: Container = with_python_base(context)
     ci_credentials = await with_installed_python_package(context, python_base_environment, CI_CREDENTIALS_SOURCE_PATH)
-
-    return ci_credentials.with_env_variable("VERSION", "dev").with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/")
+    ci_credentials = ci_credentials.with_env_variable("VERSION", "dagger_ci")
+    return ci_credentials.with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/")
 
 
 def with_alpine_packages(base_container: Container, packages_to_install: List[str]) -> Container:

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py

@@ -14,6 +14,19 @@
 
 if TYPE_CHECKING:
     from ci_connector_ops.pipelines.contexts import ConnectorContext
+    from dagger import Container
+
+
+async def mask_secrets_in_gha_logs(ci_credentials_with_downloaded_secrets: Container):
+    """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix.
+    We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger,
+    And GHA will only interpret lines starting with ::add-mask:: as secrets to mask.
+    """
+    secrets_to_mask = await ci_credentials_with_downloaded_secrets.file("/tmp/secrets_to_mask.txt").contents()
+    for secret_to_mask in secrets_to_mask.splitlines():
+        # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::"
+        # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked
+        print(f"::add-mask::{secret_to_mask}")
 
 
 async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Directory:
@@ -30,14 +43,17 @@ async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "
     secrets_path = f"/{context.connector.code_directory}/secrets"
 
     ci_credentials = await environments.with_ci_credentials(context, gsm_secret)
-    return (
+    with_downloaded_secrets = (
         ci_credentials.with_exec(["mkdir", "-p", secrets_path])
         .with_env_variable(
             "CACHEBUSTER", datetime.datetime.now().isoformat()
         )  # Secrets can be updated on GSM anytime, we can't cache this step...
         .with_exec(["ci_credentials", context.connector.technical_name, "write-to-storage"])
-        .directory(secrets_path)
     )
+    # We don't want to print secrets in the logs when running locally.
+    if context.is_ci:
+        await mask_secrets_in_gha_logs(with_downloaded_secrets)
+    return with_downloaded_secrets.directory(secrets_path)
 
 
 async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> int:

tools/ci_credentials/ci_credentials/secrets_manager.py

@@ -10,6 +10,8 @@
 from pathlib import Path
 from typing import Any, ClassVar, List, Mapping
 
+import requests
+import yaml
 from ci_common_utils import GoogleApi, Logger
 
 from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret
@@ -18,7 +20,7 @@
 
 GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",)
 
-MASK_KEY_PATTERNS = [
+DEFAULT_MASK_KEY_PATTERNS = [
     "password",
     "host",
     "user",
@@ -42,14 +44,17 @@
     "survey_",
     "appid",
     "apikey",
+    "api_key",
 ]
 
 
 class SecretsManager:
     """Loading, saving and updating all requested secrets into connector folders"""
 
+    SPEC_MASK_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"
+
     logger: ClassVar[Logger] = Logger()
-    if os.getenv("VERSION") == "dev":
+    if os.getenv("VERSION") in ["dev", "dagger_ci"]:
         base_folder = Path(os.getcwd())
     else:
         base_folder = Path("/actions-runner/_work/airbyte/airbyte")
@@ -65,6 +70,10 @@ def api(self) -> GoogleApi:
             self._api = GoogleApi(self.gsm_credentials, GSM_SCOPES)
         return self._api
 
+    @property
+    def mask_key_patterns(self) -> List[str]:
+        return self._get_spec_mask() + DEFAULT_MASK_KEY_PATTERNS
+
     def __load_gsm_secrets(self) -> List[RemoteSecret]:
         """Loads needed GSM secrets"""
         secrets = []
@@ -145,15 +154,19 @@ def mask_secrets_from_action_log(self, key, value):
         else:
             if key:
                 # regular value, check for what to mask
-                for pattern in MASK_KEY_PATTERNS:
+                for pattern in self.mask_key_patterns:
                     if re.search(pattern, key):
                         self.logger.info(f"Add mask for key: {key}")
                         for line in str(value).splitlines():
                             line = str(line).strip()
                             # don't output } and such
-                            if len(line) > 1 and not os.getenv("VERSION") == "dev":
-                                # has to be at the beginning of line for Github to notice it
-                                print(f"::add-mask::{line}")
+                            if len(line) > 1:
+                                if not os.getenv("VERSION") in ["dev", "dagger_ci"]:
+                                    # has to be at the beginning of line for Github to notice it
+                                    print(f"::add-mask::{line}")
+                                if os.getenv("VERSION") == "dagger_ci":
+                                    with open("/tmp/secrets_to_mask.txt", "a") as f:
+                                        f.write(f"{line}\n")
                         break
             # see if it's really embedded json and get those values too
             try:
@@ -271,3 +284,13 @@ def update_secrets(self, existing_secrets: List[RemoteSecret]) -> List[RemoteSec
                 new_remote_secrets.append(new_remote_secret)
                 self.logger.info(f"Updated {new_remote_secret.name} with new value")
         return new_remote_secrets
+
+    def _get_spec_mask(self) -> List[str]:
+        response = requests.get(self.SPEC_MASK_URL, allow_redirects=True)
+        if not response.ok:
+            self.logger.error(f"Failed to fetch spec mask: {response.content}")
+        try:
+            return yaml.safe_load(response.content)["properties"]
+        except Exception as e:
+            self.logger.error(f"Failed to parse spec mask: {e}")
+            return []

tools/ci_credentials/setup.py

@@ -7,7 +7,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["requests", "click~=8.1.3"]
+MAIN_REQUIREMENTS = ["requests", "click~=8.1.3", "pyyaml"]
 
 
 def local_pkg(name: str) -> str:
@@ -21,7 +21,7 @@ def local_pkg(name: str) -> str:
 TEST_REQUIREMENTS = ["requests-mock", "pytest"]
 
 setup(
-    version="1.0.1",
+    version="1.1.0",
     name="ci_credentials",
     description="CLI tooling to read and manage GSM secrets",
     author="Airbyte",