Remove release stage references in favor of support level or ab_inter…

…nal (#29232)

* First pass on removing release stage references

* Automated Commit - Format and Process Resources Changes

* Fix metadata tests

* Fix qa engine tests

* Fix pipeline tests

* Update registry report

* Fix support-level arg

* Add dummy change

* Deal with --allow_community

* Improve log message

* Add docs

* Revert "Add dummy change"

This reverts commit 1861b95.

* Code review comments

* Run format

* Install connector ops to qa-engine

* Fix CI test

* REVERT ME

* Revert "REVERT ME"

This reverts commit dce9b8e.

---------

Co-authored-by: bnchrch <bnchrch@users.noreply.github.com>

.github/workflows/connectors_nightly_build.yml

@@ -11,14 +11,14 @@ on:
         default: conn-nightly-xlarge-runner
         required: true
       test-connectors-options:
-        default: --concurrency=5 --release-stage=generally_available --release-stage=beta
+        default: --concurrency=5 --support-level=certified
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for GA and Beta connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for GA and Beta connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
     timeout-minutes: 720 # 12 hours
     runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}
     steps:
@@ -41,4 +41,4 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           git_branch: ${{ steps.extract_branch.outputs.branch }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --release-stage=generally_available --release-stage=beta' }} test"
+          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test"

.github/workflows/connectors_weekly_build.yml

@@ -11,14 +11,14 @@ on:
         default: conn-nightly-xlarge-runner
         required: true
       test-connectors-options:
-        default: --concurrency=3 --release-stage=alpha
+        default: --concurrency=3 --support-level=community
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Alpha connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
 
 jobs:
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Alpha connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}"
     timeout-minutes: 8640 # 6 days
     runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}
     steps:
@@ -41,4 +41,4 @@ jobs:
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           git_branch: ${{ steps.extract_branch.outputs.branch }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=3 --release-stage=alpha' }} test"
+          subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=3 --support-level=community' }} test"

airbyte-ci/connectors/ci_credentials/tests/test_secrets_manager.py

@@ -20,6 +20,7 @@ def matchers():
         "addVersion": re.compile("https://secretmanager.googleapis.com/v1/.+:addVersion"),
         "access": re.compile("https://secretmanager.googleapis.com/v1/.+/1:access"),
         "disable": re.compile("https://secretmanager.googleapis.com/v1/.+:disable"),
+        "spec_secret_mask": re.compile("https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"),
     }
 
 
@@ -92,6 +93,7 @@ def test_read(matchers, connector_name, gsm_secrets, expected_secrets):
         m.post(matchers["secrets"], json={"name": "<fake_name>"})
         m.get(matchers["versions"], versions_response_list)
         m.get(matchers["access"], secrets_response_list)
+        m.get(matchers["spec_secret_mask"], json={"spec_secret_mask": "test"})
 
         secrets = manager.read_from_gsm()
         assert secrets == expected_secrets

airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py

@@ -9,7 +9,7 @@
 
 from connector_ops import utils
 
-RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING = {"generally_available": "high"}
+
 BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"}
 TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"}
 GA_BYPASS_REASON_REVIEWERS = {"connector-operations"}
@@ -30,51 +30,48 @@ def find_connectors_with_bad_strictness_level() -> List[utils.Connector]:
     connectors_with_bad_strictness_level = []
     changed_connector = utils.get_changed_connectors(destination=False, third_party=False)
     for connector in changed_connector:
-        expected_test_strictness_level = RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING.get(connector.release_stage)
-        can_check_strictness_level = all(
-            [item is not None for item in [connector.release_stage, expected_test_strictness_level, connector.acceptance_test_config]]
-        )
-        if can_check_strictness_level:
+        check_for_high_strictness = connector.acceptance_test_config is not None and connector.requires_high_test_strictness_level
+        if check_for_high_strictness:
             try:
-                assert connector.acceptance_test_config.get("test_strictness_level") == expected_test_strictness_level
+                assert connector.acceptance_test_config.get("test_strictness_level") == "high"
             except AssertionError:
                 connectors_with_bad_strictness_level.append(connector)
     return connectors_with_bad_strictness_level
 
 
-def find_changed_ga_connectors() -> Set[utils.Connector]:
-    """Find GA connectors modified on the current branch.
+def find_changed_important_connectors() -> Set[utils.Connector]:
+    """Find important connectors modified on the current branch.
 
     Returns:
         Set[utils.Connector]: The set of GA connectors that were modified on the current branch.
     """
     changed_connectors = utils.get_changed_connectors(destination=False, third_party=False)
-    return {connector for connector in changed_connectors if connector.release_stage == "generally_available"}
+    return {connector for connector in changed_connectors if connector.is_important_connector}
 
 
-def get_ga_bypass_reason_changes() -> Set[utils.Connector]:
-    """Find GA connectors that have modified bypass_reasons.
+def get_bypass_reason_changes() -> Set[utils.Connector]:
+    """Find connectors that have modified bypass_reasons.
 
     Returns:
         Set[str]: Set of connector names e.g {"source-github"}: The set of GA connectors that have changed bypass_reasons.
     """
     bypass_reason_changes = utils.get_changed_acceptance_test_config(diff_regex="bypass_reason")
-    return bypass_reason_changes.intersection(find_changed_ga_connectors())
+    return bypass_reason_changes.intersection(find_changed_important_connectors())
 
 
 def find_mandatory_reviewers() -> List[Union[str, Dict[str, List]]]:
-    ga_connector_changes = find_changed_ga_connectors()
+    important_connector_changes = find_changed_important_connectors()
     backward_compatibility_changes = utils.get_changed_acceptance_test_config(diff_regex="disable_for_version")
     test_strictness_level_changes = utils.get_changed_acceptance_test_config(diff_regex="test_strictness_level")
-    ga_bypass_reason_changes = get_ga_bypass_reason_changes()
+    ga_bypass_reason_changes = get_bypass_reason_changes()
 
     if backward_compatibility_changes:
         return [{"any-of": list(BACKWARD_COMPATIBILITY_REVIEWERS)}]
     if test_strictness_level_changes:
         return [{"any-of": list(TEST_STRICTNESS_LEVEL_REVIEWERS)}]
     if ga_bypass_reason_changes:
         return [{"any-of": list(GA_BYPASS_REASON_REVIEWERS)}]
-    if ga_connector_changes:
+    if important_connector_changes:
         return list(GA_CONNECTOR_REVIEWERS)
     return []
 
@@ -83,7 +80,7 @@ def check_test_strictness_level():
     connectors_with_bad_strictness_level = find_connectors_with_bad_strictness_level()
     if connectors_with_bad_strictness_level:
         logging.error(
-            f"The following GA connectors must enable high test strictness level: {connectors_with_bad_strictness_level}. Please check this documentation for details: https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/#strictness-level"
+            f"The following connectors must enable high test strictness level: {connectors_with_bad_strictness_level}. Please check this documentation for details: https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/#strictness-level"
         )
         sys.exit(1)
     else:

airbyte-ci/connectors/connector_ops/connector_ops/allowed_hosts_checks.py

@@ -8,15 +8,36 @@
 
 from connector_ops import utils
 
-RELEASE_STAGES_TO_CHECK = ["generally_available", "beta"]
+ALLOWED_HOST_THRESHOLD = {
+    "sl": 200,
+    "ql": 300,
+}
+
+
+def _requires_allowed_hosts(connector: utils.Connector) -> bool:
+    """Check if a connector requires allowed hosts.
+
+    Args:
+        connector (utils.Connector): The connector to check.
+
+    Returns:
+        bool: True if the connector requires allowed hosts, False otherwise.
+    """
+    if connector.ab_internal_sl >= ALLOWED_HOST_THRESHOLD["sl"]:
+        return True
+
+    if connector.ab_internal_ql >= ALLOWED_HOST_THRESHOLD["ql"]:
+        return True
+
+    return False
 
 
 def get_connectors_missing_allowed_hosts() -> List[utils.Connector]:
     connectors_missing_allowed_hosts: List[utils.Connector] = []
     changed_connectors = utils.get_changed_connectors(destination=False, third_party=False)
 
     for connector in changed_connectors:
-        if connector.release_stage in RELEASE_STAGES_TO_CHECK:
+        if _requires_allowed_hosts(connector):
             missing = not connector_has_allowed_hosts(connector)
             if missing:
                 connectors_missing_allowed_hosts.append(connector)
@@ -31,7 +52,7 @@ def connector_has_allowed_hosts(connector: utils.Connector) -> bool:
 def check_allowed_hosts():
     connectors_missing_allowed_hosts = get_connectors_missing_allowed_hosts()
     if connectors_missing_allowed_hosts:
-        logging.error(f"The following {RELEASE_STAGES_TO_CHECK} connectors must include allowedHosts: {connectors_missing_allowed_hosts}")
+        logging.error(f"The following connectors must include allowedHosts: {connectors_missing_allowed_hosts}")
         sys.exit(1)
     else:
         sys.exit(0)

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -11,6 +11,7 @@
 from glob import glob
 from pathlib import Path
 from typing import List, Optional, Set, Tuple, Union
+from pydash.objects import get
 
 import git
 import requests
@@ -45,6 +46,11 @@ def download_catalog(catalog_url):
 METADATA_FILE_NAME = "metadata.yaml"
 ICON_FILE_NAME = "icon.svg"
 
+IMPORTANT_CONNECTOR_THRESHOLDS = {
+    "sl": 300,
+    "ql": 400,
+}
+
 
 class ConnectorInvalidNameError(Exception):
     pass
@@ -273,8 +279,72 @@ def name_from_metadata(self) -> Optional[str]:
         return self.metadata.get("name") if self.metadata else None
 
     @property
-    def release_stage(self) -> Optional[str]:
-        return self.metadata.get("releaseStage") if self.metadata else None
+    def support_level(self) -> Optional[str]:
+        return self.metadata.get("supportLevel") if self.metadata else None
+
+    @property
+    def ab_internal_sl(self) -> int:
+        """Airbyte Internal Field.
+
+        More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9
+
+        Returns:
+            int: The value
+        """
+        default_value = 100
+        sl_value = get(self.metadata, "ab_internal.sl")
+
+        if sl_value is None:
+            logging.warning(
+                f"Connector {self.technical_name} does not have a `ab_internal.sl` defined in metadata.yaml. Defaulting to {default_value}"
+            )
+            return default_value
+
+        return sl_value
+
+    @property
+    def ab_internal_ql(self) -> int:
+        """Airbyte Internal Field.
+
+        More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9
+
+        Returns:
+            int: The value
+        """
+        default_value = 100
+        ql_value = get(self.metadata, "ab_internal.ql")
+
+        if ql_value is None:
+            logging.warning(
+                f"Connector {self.technical_name} does not have a `ab_internal.ql` defined in metadata.yaml. Defaulting to {default_value}"
+            )
+            return default_value
+
+        return ql_value
+
+    @property
+    def is_important_connector(self) -> bool:
+        """Check if a connector qualifies as an important connector.
+
+        Returns:
+            bool: True if the connector is a high value connector, False otherwise.
+        """
+        if self.ab_internal_sl >= IMPORTANT_CONNECTOR_THRESHOLDS["sl"]:
+            return True
+
+        if self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"]:
+            return True
+
+        return False
+
+    @property
+    def requires_high_test_strictness_level(self) -> bool:
+        """Check if a connector requires high strictness CAT tests.
+
+        Returns:
+            bool: True if the connector requires high test strictness level, False otherwise.
+        """
+        return self.is_important_connector()
 
     @property
     def allowed_hosts(self) -> Optional[List[str]]:
@@ -366,3 +436,13 @@ def get_all_connectors_in_repo() -> Set[Connector]:
         for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
         if SCAFFOLD_CONNECTOR_GLOB not in metadata_file
     }
+
+
+class ConnectorTypeEnum(str, Enum):
+    source = "source"
+    destination = "destination"
+
+
+class SupportLevelEnum(str, Enum):
+    certified = "certified"
+    community = "community"