-
Notifications
You must be signed in to change notification settings - Fork 3.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Remove release stage references in favor of support level or ab_internal #29232
Changes from all commits
320d982
473d01e
f714830
0e0c241
fd65b29
54e79c5
e7ddacc
abb3046
1861b95
4828b13
692842d
205fa6d
31aa865
0c7c6a6
ef3c687
00f0361
c571681
6c0260c
dce9b8e
9538097
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,15 +8,36 @@ | |
|
||
from connector_ops import utils | ||
|
||
RELEASE_STAGES_TO_CHECK = ["generally_available", "beta"] | ||
ALLOWED_HOST_THRESHOLD = { | ||
"sl": 200, | ||
"ql": 300, | ||
} | ||
|
||
|
||
def _requires_allowed_hosts(connector: utils.Connector) -> bool: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same idea as here https://github.com/airbytehq/airbyte/pull/29232/files#r1287793875 |
||
"""Check if a connector requires allowed hosts. | ||
Args: | ||
connector (utils.Connector): The connector to check. | ||
Returns: | ||
bool: True if the connector requires allowed hosts, False otherwise. | ||
""" | ||
if connector.ab_internal_sl >= ALLOWED_HOST_THRESHOLD["sl"]: | ||
return True | ||
|
||
if connector.ab_internal_ql >= ALLOWED_HOST_THRESHOLD["ql"]: | ||
return True | ||
|
||
return False | ||
|
||
|
||
def get_connectors_missing_allowed_hosts() -> List[utils.Connector]: | ||
connectors_missing_allowed_hosts: List[utils.Connector] = [] | ||
changed_connectors = utils.get_changed_connectors(destination=False, third_party=False) | ||
|
||
for connector in changed_connectors: | ||
if connector.release_stage in RELEASE_STAGES_TO_CHECK: | ||
if _requires_allowed_hosts(connector): | ||
missing = not connector_has_allowed_hosts(connector) | ||
if missing: | ||
connectors_missing_allowed_hosts.append(connector) | ||
|
@@ -31,7 +52,7 @@ def connector_has_allowed_hosts(connector: utils.Connector) -> bool: | |
def check_allowed_hosts(): | ||
connectors_missing_allowed_hosts = get_connectors_missing_allowed_hosts() | ||
if connectors_missing_allowed_hosts: | ||
logging.error(f"The following {RELEASE_STAGES_TO_CHECK} connectors must include allowedHosts: {connectors_missing_allowed_hosts}") | ||
logging.error(f"The following connectors must include allowedHosts: {connectors_missing_allowed_hosts}") | ||
sys.exit(1) | ||
else: | ||
sys.exit(0) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,6 +11,7 @@ | |
from glob import glob | ||
from pathlib import Path | ||
from typing import List, Optional, Set, Tuple, Union | ||
from pydash.objects import get | ||
|
||
import git | ||
import requests | ||
|
@@ -45,6 +46,11 @@ def download_catalog(catalog_url): | |
METADATA_FILE_NAME = "metadata.yaml" | ||
ICON_FILE_NAME = "icon.svg" | ||
|
||
IMPORTANT_CONNECTOR_THRESHOLDS = { | ||
"sl": 300, | ||
"ql": 400, | ||
} | ||
|
||
|
||
class ConnectorInvalidNameError(Exception): | ||
pass | ||
|
@@ -273,8 +279,72 @@ def name_from_metadata(self) -> Optional[str]: | |
return self.metadata.get("name") if self.metadata else None | ||
|
||
@property | ||
def release_stage(self) -> Optional[str]: | ||
return self.metadata.get("releaseStage") if self.metadata else None | ||
def support_level(self) -> Optional[str]: | ||
return self.metadata.get("supportLevel") if self.metadata else None | ||
|
||
@property | ||
def ab_internal_sl(self) -> int: | ||
"""Airbyte Internal Field. | ||
More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added a new page to notion since our wiki is now deprecated |
||
Returns: | ||
int: The value | ||
""" | ||
default_value = 100 | ||
sl_value = get(self.metadata, "ab_internal.sl") | ||
|
||
if sl_value is None: | ||
logging.warning( | ||
f"Connector {self.technical_name} does not have a `ab_internal.sl` defined in metadata.yaml. Defaulting to {default_value}" | ||
) | ||
return default_value | ||
|
||
return sl_value | ||
|
||
@property | ||
def ab_internal_ql(self) -> int: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These should already be defined but putting defaults here in case. These should be converted to hard errors on the introduction of metadata v2 where |
||
"""Airbyte Internal Field. | ||
More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9 | ||
Returns: | ||
int: The value | ||
""" | ||
default_value = 100 | ||
ql_value = get(self.metadata, "ab_internal.ql") | ||
|
||
if ql_value is None: | ||
logging.warning( | ||
f"Connector {self.technical_name} does not have a `ab_internal.ql` defined in metadata.yaml. Defaulting to {default_value}" | ||
) | ||
return default_value | ||
|
||
return ql_value | ||
|
||
@property | ||
def is_important_connector(self) -> bool: | ||
"""Check if a connector qualifies as an important connector. | ||
Returns: | ||
bool: True if the connector is a high value connector, False otherwise. | ||
""" | ||
if self.ab_internal_sl >= IMPORTANT_CONNECTOR_THRESHOLDS["sl"]: | ||
return True | ||
|
||
if self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"]: | ||
return True | ||
|
||
return False | ||
|
||
@property | ||
def requires_high_test_strictness_level(self) -> bool: | ||
"""Check if a connector requires high strictness CAT tests. | ||
Returns: | ||
bool: True if the connector requires high test strictness level, False otherwise. | ||
""" | ||
return self.is_important_connector() | ||
|
||
@property | ||
def allowed_hosts(self) -> Optional[List[str]]: | ||
|
@@ -366,3 +436,13 @@ def get_all_connectors_in_repo() -> Set[Connector]: | |
for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) | ||
if SCAFFOLD_CONNECTOR_GLOB not in metadata_file | ||
} | ||
|
||
|
||
class ConnectorTypeEnum(str, Enum): | ||
source = "source" | ||
destination = "destination" | ||
|
||
|
||
class SupportLevelEnum(str, Enum): | ||
certified = "certified" | ||
community = "community" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All the metadata are up to date to match the same subset of connector nightly build originally targets?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
They dont match the same subset unfortunately. There will likely be a bit of a mix up