add base-normalization to dependency checker action (#19313)

* Add normalization to list of modules to check dependencies for

* Remove now unneeded empty lists

* Refactor how gradle files are retrieved

* Let method set the dict key for module

* Rename CONNECTOR_PATH, ensure that path for get_gradle_file_for_path is always a folder

* Add typing on potentially confusing method

* Make comment template clearer

* Move modules into details

* Auto-close sources and destinations, add emoji and count for other modules

* Rename function to get dependent modules

.github/comment_templates/connector_dependency_template.md

@@ -4,7 +4,7 @@
 
 NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do the following as needed:
 - Run integration tests
-- Bump connector version
+- Bump connector or module version
 - Add changelog
 - Publish the new version
 
@@ -38,7 +38,16 @@ NOTE ⚠️ Changes in this PR affect the following connectors. Make sure to do
 
 </details>
 
-{others}
+<details open>
+<summary>
+
+### {other_status_summary} Other Modules ({num_others})
+
+</summary>
+
+{others_rows}
+
+</details>
 
 <details>
 

tools/bin/ci_check_dependency.py

@@ -2,8 +2,10 @@
 import os
 import os.path
 import yaml
+from typing import Any, Dict, Text
 
-CONNECTOR_PATH = "./airbyte-integrations/connectors/"
+CONNECTORS_PATH = "./airbyte-integrations/connectors/"
+NORMALIZATION_PATH = "./airbyte-integrations/bases/base-normalization/"
 DOC_PATH = "docs/integrations/"
 SOURCE_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/source_definitions.yaml"
 DESTINATION_DEFINITIONS_PATH = "./airbyte-config/init/src/main/resources/seed/destination_definitions.yaml"
@@ -32,14 +34,18 @@ def main():
     all_connectors = get_all_connectors()
 
     # Getting all build.gradle file
-    all_build_gradle_files = get_connectors_gradle_files(all_connectors)
+    build_gradle_files = {}
+    for connector in all_connectors:
+        connector_path = CONNECTORS_PATH + connector + "/"
+        build_gradle_files.update(get_gradle_file_for_path(connector_path))
+    build_gradle_files.update(get_gradle_file_for_path(NORMALIZATION_PATH))
 
     # Try to find dependency in build.gradle file
-    depended_connectors = list(set(get_depended_connectors(changed_modules, all_build_gradle_files)))
+    dependent_modules = list(set(get_dependent_modules(changed_modules, build_gradle_files)))
 
     # Create comment body to post on pull request
-    if depended_connectors:
-        write_report(depended_connectors)
+    if dependent_modules:
+        write_report(dependent_modules)
 
 
 def get_changed_files(path):
@@ -61,17 +67,16 @@ def get_changed_modules(changed_files):
 
 
 def get_all_connectors():
-    walk = os.walk(CONNECTOR_PATH)
+    walk = os.walk(CONNECTORS_PATH)
     return [connector for connector in next(walk)[1]]
 
 
-def get_connectors_gradle_files(all_connectors):
-    all_build_gradle_files = {}
-    for connector in all_connectors:
-        build_gradle_path = CONNECTOR_PATH + connector + "/"
-        build_gradle_file = find_file("build.gradle", build_gradle_path)
-        all_build_gradle_files[connector] = build_gradle_file
-    return all_build_gradle_files
+def get_gradle_file_for_path(path: str) -> Dict[Text, Any]:
+    if not path.endswith("/"):
+        path = path + "/"
+    build_gradle_file = find_file("build.gradle", path)
+    module = path.split("/")[-2]
+    return {module: build_gradle_file}
 
 
 def find_file(name, path):
@@ -80,20 +85,20 @@ def find_file(name, path):
             return os.path.join(root, name)
 
 
-def get_depended_connectors(changed_modules, all_build_gradle_files):
-    depended_connectors = []
+def get_dependent_modules(changed_modules, all_build_gradle_files):
+    dependent_modules = []
     for changed_module in changed_modules:
-        for connector, gradle_file in all_build_gradle_files.items():
+        for module, gradle_file in all_build_gradle_files.items():
             if gradle_file is None:
                 continue
             with open(gradle_file) as file:
                 if changed_module in file.read():
-                    depended_connectors.append(connector)
-    return depended_connectors
+                    dependent_modules.append(module)
+    return dependent_modules
 
 
 def get_connector_version(connector):
-    with open(f"{CONNECTOR_PATH}/{connector}/Dockerfile") as f:
+    with open(f"{CONNECTORS_PATH}/{connector}/Dockerfile") as f:
         for line in f:
             if "io.airbyte.version" in line:
                 return line.split("=")[1].strip()
@@ -126,6 +131,7 @@ def get_connector_changelog_status(connector, version):
                 return "✅"
     return "❌<br/>(changelog missing)"
 
+
 def as_bulleted_markdown_list(items):
     text = ""
     for item in items:
@@ -174,38 +180,34 @@ def write_report(depended_connectors):
     with open(COMMENT_TEMPLATE_PATH, "r") as f:
         template = f.read()
 
-    source_definitions = []
-    destination_definitions = []
     with open(SOURCE_DEFINITIONS_PATH, 'r') as stream:
         source_definitions = yaml.safe_load(stream)
     with open(DESTINATION_DEFINITIONS_PATH, 'r') as stream:
         destination_definitions = yaml.safe_load(stream)
 
-    others_md = ""
-    if affected_others:
-        others_md += "The following were also affected:\n"
-        others_md += as_bulleted_markdown_list(affected_others)
-
     affected_sources.sort()
     affected_destinations.sort()
     affected_others.sort()
 
     source_rows = as_markdown_table_rows(affected_sources, source_definitions)
     destination_rows = as_markdown_table_rows(affected_destinations, destination_definitions)
 
+    other_status_summary = "✅" if len(affected_others) == 0 else "👀"
     source_status_summary = get_status_summary(source_rows)
     destination_status_summary = get_status_summary(destination_rows)
 
     comment = template.format(
-        source_open="open" if source_status_summary == "❌" else "closed",
-        destination_open="open" if destination_status_summary == "❌" else "closed",
+        source_open="closed",  # if source_status_summary == "❌" else "closed", (see #19451)
+        destination_open="closed",  # if destination_status_summary == "❌" else "closed", (see #19451)
         source_status_summary=source_status_summary,
         destination_status_summary=destination_status_summary,
+        other_status_summary=other_status_summary,
         source_rows=source_rows,
         destination_rows=destination_rows,
-        others=others_md,
+        others_rows=as_bulleted_markdown_list(affected_others),
         num_sources=len(affected_sources),
-        num_destinations=len(affected_destinations)
+        num_destinations=len(affected_destinations),
+        num_others=len(affected_others),
     )
 
     with open("comment_body.md", "w") as f: