From 04cb57a14bc2cca347c6c9a3c48c8edb94fd16be Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 20 Feb 2024 09:12:09 +0100 Subject: [PATCH] remove qa_checks.py --- .../connector_ops/connector_ops/qa_checks.py | 320 ------------------ .../connectors/connector_ops/pyproject.toml | 2 +- .../connector_ops/tests/test_qa_checks.py | 274 --------------- .../testing-connectors/README.md | 2 +- 4 files changed, 2 insertions(+), 596 deletions(-) delete mode 100644 airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py delete mode 100644 airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py deleted file mode 100644 index 99bd8e1786b0ee..00000000000000 --- a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py +++ /dev/null @@ -1,320 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys -from pathlib import Path -from typing import Callable, Iterable, Optional, Set, Tuple - -from connector_ops.utils import Connector, ConnectorLanguage -from pydash.objects import get - - -def check_migration_guide(connector: Connector) -> bool: - """Check if a migration guide is available for the connector if a breaking change was introduced.""" - - breaking_changes = get(connector.metadata, "releases.breakingChanges") - if not breaking_changes: - return True - - migration_guide_file_path = connector.migration_guide_file_path - migration_guide_exists = migration_guide_file_path is not None and migration_guide_file_path.exists() - if not migration_guide_exists: - print( - f"Migration guide file is missing for {connector.name}. Please create a migration guide at {connector.migration_guide_file_path}" - ) - return False - - # Check that the migration guide begins with # {connector name} Migration Guide - expected_title = f"# {connector.name_from_metadata} Migration Guide" - expected_version_header_start = "## Upgrading to " - with open(migration_guide_file_path) as f: - first_line = f.readline().strip() - if not first_line == expected_title: - print( - f"Migration guide file for {connector.technical_name} does not start with the correct header. Expected '{expected_title}', got '{first_line}'" - ) - return False - - # Check that the migration guide contains a section for each breaking change key ## Upgrading to {version} - # Note that breaking change is a dict where the version is the key - # Note that the migration guide must have the sections in order of the version descending - # 3.0.0, 2.0.0, 1.0.0, etc - # This means we have to record the headings in the migration guide and then check that they are in order - # We also have to check that the headings are in the breaking changes dict - - ordered_breaking_changes = sorted(breaking_changes.keys(), reverse=True) - ordered_expected_headings = [f"{expected_version_header_start}{version}" for version in ordered_breaking_changes] - - ordered_heading_versions = [] - for line in f: - stripped_line = line.strip() - if stripped_line.startswith(expected_version_header_start): - version = stripped_line.replace(expected_version_header_start, "") - ordered_heading_versions.append(version) - - if ordered_breaking_changes != ordered_heading_versions: - print(f"Migration guide file for {connector.name} has incorrect version headings.") - print("Check for missing, extra, or misordered headings, or headers with typos.") - print(f"Expected headings: {ordered_expected_headings}") - return False - - return True - - -def check_documentation_file_exists(connector: Connector) -> bool: - """Check if a markdown file with connector documentation is available - in docs/integrations/s/.md - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether a documentation file was found. - """ - file_path = connector.documentation_file_path - - return file_path is not None and file_path.exists() - - -def check_documentation_follows_guidelines(connector: Connector) -> bool: - """Documentation guidelines are defined here https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw""" - follows_guidelines = True - with open(connector.documentation_file_path) as f: - doc_lines = [line.lower() for line in f.read().splitlines()] - if not doc_lines[0].startswith("# "): - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - # We usually don't have a metadata if the connector is not published. - if connector.metadata: - if doc_lines[0].strip() != f"# {connector.metadata['name'].lower()}": - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - elif not doc_lines[0].startswith("# "): - print("The connector name is not used as the main header in the documentation.") - follows_guidelines = False - - expected_sections = ["## Prerequisites", "## Setup guide", "## Supported sync modes", "## Supported streams", "## Changelog"] - - for expected_section in expected_sections: - if expected_section.lower() not in doc_lines: - print(f"Connector documentation is missing a '{expected_section.replace('#', '').strip()}' section.") - follows_guidelines = False - return follows_guidelines - - -def check_changelog_entry_is_updated(connector: Connector) -> bool: - """Check that the changelog entry is updated for the latest connector version - in docs/integrations//.md - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether a the changelog is up to date. - """ - if not check_documentation_file_exists(connector): - return False - with open(connector.documentation_file_path) as f: - after_changelog = False - for line in f: - if "# changelog" in line.lower(): - after_changelog = True - if after_changelog and connector.version in line: - return True - return False - - -def check_connector_icon_is_available(connector: Connector) -> bool: - """Check an SVG icon exists for a connector in - in airbyte-config-oss/init-oss/src/main/resources/icons/.svg - - Args: - connector (Connector): a Connector dataclass instance. - - Returns: - bool: Wether an icon exists for this connector. - """ - return connector.icon_path.exists() - - -def read_all_files_in_directory( - directory: Path, ignored_directories: Optional[Set[str]] = None, ignored_filename_patterns: Optional[Set[str]] = None -) -> Iterable[Tuple[str, str]]: - ignored_directories = ignored_directories if ignored_directories is not None else {} - ignored_filename_patterns = ignored_filename_patterns if ignored_filename_patterns is not None else {} - - for path in directory.rglob("*"): - ignore_directory = any([ignored_directory in path.parts for ignored_directory in ignored_directories]) - ignore_filename = any([path.match(ignored_filename_pattern) for ignored_filename_pattern in ignored_filename_patterns]) - ignore = ignore_directory or ignore_filename - if path.is_file() and not ignore: - try: - for line in open(path, "r"): - yield path, line - except UnicodeDecodeError: - print(f"{path} could not be decoded as it is not UTF8.") - continue - - -IGNORED_DIRECTORIES_FOR_HTTPS_CHECKS = { - ".venv", - "tests", - "unit_tests", - "integration_tests", - "build", - "source-file", - ".pytest_cache", - "acceptance_tests_logs", - ".hypothesis", -} - -IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = { - "*Test.java", - "*.jar", - "*.pyc", - "*.gz", - "*.svg", - "expected_records.jsonl", - "expected_records.json", -} -IGNORED_URLS_PREFIX = { - "http://json-schema.org", - "http://localhost", -} - - -def is_comment(line: str, file_path: Path): - language_comments = { - ".py": "#", - ".yml": "#", - ".yaml": "#", - ".java": "//", - ".md": "