Add GH Actions for dbt installation tests (#71)

fetch-container-tags:
- Added initial action implementation;
- Created `README` file;

fetch-repo-branches:
- Added initial action implementation;
- Created `README` file;

test-dbt-installation-docker.yml:
- GitHub Script action replaced with `fetch-container-tags` action

test-dbt-installation-source.yml:
- GitHub Script action replaced with `fetch-repo-branches` action

.github/workflows/test-dbt-installation-docker.yml

@@ -61,42 +61,24 @@ jobs:
     runs-on: ubuntu-latest
 
     outputs:
-      latest-tags: ${{ steps.get-latest-tags.outputs.result }}
+      latest-tags: ${{ steps.get-latest-tags.outputs.container-tags }}
 
     steps:
       - name: "Fetch ${{ inputs.package_name }} Tags From ${{ env.GITHUB_PACKAGES_LINK }}"
-        # Description: Fetch tags for specific container and filter out `latest` tags
-        # GH API doc: https://docs.github.com/en/rest/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-an-organization
-        uses: actions/github-script@v6
+        uses: dbt-labs/actions/fetch-container-tags@v1.1.1
         id: get-latest-tags
         with:
+          package_name: ${{ inputs.package_name }}
+          organization: "dbt-labs"
+          pat: ${{ secrets.GITHUB_TOKEN }}
+          regex: "latest$"
+          perform_match_method: "search"
           retries: 3
-          script: |
-            try {
-                const response = await github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg({
-                    package_type: 'container',
-                    package_name: '${{ inputs.package_name }}',
-                    org: 'dbt-labs',
-                })
-                let tags = [];
-                if (response.data) {
-                    const package_data = response.data;
-                    for (const [key, value] of Object.entries(package_data)) {
-                        if (value.metadata.container.tags) {
-                            tags = tags.concat(value.metadata.container.tags)
-                        }
-                    }
-                    tags = tags.filter(t => t.endsWith('latest'))
-                }
-                return tags
-            } catch (error) {
-                core.setFailed(error.message);
-            }
 
       - name: "[ANNOTATION] ${{ inputs.package_name }} - tags to test"
         run: |
           title="${{ inputs.package_name }} - tags to test"
-          message="The workflow will run tests for the following tags of the ${{ inputs.package_name }} image: ${{ steps.get-latest-tags.outputs.result }}"
+          message="The workflow will run tests for the following tags of the ${{ inputs.package_name }} image: ${{ steps.get-latest-tags.outputs.container-tags }}"
           echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
 
   docker-installation-test:

.github/workflows/test-dbt-installation-notify-job-statuses.yml

@@ -77,7 +77,7 @@ jobs:
             core.debug(files_list);
 
             files_list.map(file => {
-                if (file.includes(${{ inputs.package_name }})) {
+                if (file.includes("${{ inputs.package_name }}")) {
                   const buffer = fs.readFileSync(`${artifact_folder}/${file}`);
                   const jobs_status = buffer.toString().trim().replace(/['"]+/g, '');
 

.github/workflows/test-dbt-installation-source.yml

@@ -58,41 +58,25 @@ jobs:
     runs-on: ubuntu-latest
 
     outputs:
-      latest-branches: ${{ steps.get-latest-branches.outputs.result }}
+      latest-branches: ${{ steps.get-latest-branches.outputs.repo-branches }}
 
     steps:
-      # Description: Fetch protected branches metadata for specific repo and filter out `latest` branches
-      # GH API doc: https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches
-      - name: "Fetch ${{ inputs.package_name }} Protected Branches Metadata"
-        uses: actions/github-script@v6
+      - name: "Fetch dbt-core Latest Branches"
+        uses: dbt-labs/actions/fetch-repo-branches@v1.1.1
         id: get-latest-branches
         with:
+          repo_name: ${{ inputs.package_name }}
+          organization: "dbt-labs"
+          pat: ${{ secrets.GITHUB_TOKEN }}
+          fetch_protected_branches_only: true
+          regex: "^1.[0-9]+.latest$"
+          perform_match_method: "match"
           retries: 3
-          script: |
-            try {
-                const response = await github.rest.repos.listBranches({
-                    repo: '${{ inputs.package_name }}',
-                    owner: 'dbt-labs',
-                    protected: true,
-                })
-                let branches = []
-                if (response.data) {
-                    const branches_data = response.data
-                    for (const [key, value] of Object.entries(branches_data)) {
-                        branches = branches.concat(value.name)
-                    }
-                    const re_latest_branch = new RegExp('^1.*.latest')
-                    branches = branches.filter(b => re_latest_branch.test(b))
-                }
-                return branches
-            } catch (error) {
-                core.setFailed(error.message)
-            }
 
       - name: "[ANNOTATION] ${{ inputs.package_name }} - branches to test"
         run: |
           title="${{ inputs.package_name }} - branches to test"
-          message="The workflow will run tests for the following branches of the ${{ inputs.package_name }} repo: ${{ steps.get-latest-branches.outputs.result }}"
+          message="The workflow will run tests for the following branches of the ${{ inputs.package_name }} repo: ${{ steps.get-latest-branches.outputs.repo-branches }}"
           echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
 
   source-installation-test:

fetch-container-tags/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-slim AS builder
+ADD . /app
+WORKDIR /app
+
+# We are installing a dependency here directly into our app source dir
+RUN pip install --target=/app setuptools requests
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/distroless
+FROM gcr.io/distroless/python3-debian10
+COPY --from=builder /app /app
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]

fetch-container-tags/README.md

@@ -0,0 +1,65 @@
+# Fetch Container Tags
+
+A [GitHub Action](https://github.com/features/actions) for fetching container tags from [GitHub packages](https://ghcr.io).
+
+Example usage:
+
+```yaml
+name: Example Workflow for Fetch Container Tags
+on: push
+jobs:
+  fetch-latest-tags:
+    runs-on: ubuntu-latest
+
+    outputs:
+      latest-tags: ${{ steps.fetch-latest-tags.outputs.container-tags }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Fetch dbt-postgres Container Tags"
+        id: get-latest-tags
+        uses: dbt-labs/actions/fetch-container-tags
+        with:
+          package_name: "dbt-postgres"
+          organization: "dbt-labs"
+          pat: ${{ secrets.GITHUB_TOKEN }}
+          regex: "latest$"
+          perform_match_method: "search"
+          retries: 3
+
+      - name: "Display Container Tags"
+        run: |
+          echo container latest tags: ${{ steps.fetch-latest-tags.outputs.container-tags }}
+    
+  dynamic-matrix:
+    runs-on: ubuntu-latest
+    needs: fetch-latest-tags
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tag: ${{ fromJSON(needs.fetch-latest-tags.outputs.latest-tags) }}
+
+    steps:
+      - name: "Display Tag Name"
+        run: |
+          echo container tag: ${{ matrix.tag }}
+```
+
+### Inputs
+
+| Property             | Required | Default        | Description                                                   |
+| -------------------- | -------- | -------------- | ------------------------------------------------------------- |
+| package_name         | yes      | -              | Container name                                                |
+| organization         | yes      | -              | Organization that owns the package                            |
+| pat                  | yes      | -              | PAT for fetch request                                         |
+| regex                | no       | `empty string` | Filter container tags                                         |
+| perform_match_method | no       | `match`        | Select which method use to filter tags (search/match/findall) |
+| retries              | no       | `3`            | Retries for fetch request                                     |
+
+### Outputs
+
+| Property       | Example                                                              | Description            |
+| -------------- | -------------------------------------------------------------------- | ---------------------- |
+| container-tags | `['1.2.latest', 'latest', '1.3.latest', '1.1.latest', '1.0.latest']` | List of container tags |

fetch-container-tags/action.yml

@@ -0,0 +1,29 @@
+name: "Fetch Container Tags"
+description: "Gets Container Tags From GitHub Packages"
+inputs:
+  package_name:
+    description: "Package name"
+    required: true
+  organization:
+    description: "GitHub organization where package is stored"
+    required: true
+  pat:
+    description: "Personal access token"
+    required: true
+  regex:
+    description: "Regexp will be applied to fetch request result"
+    required: true
+  perform_match_method:
+    description: "Set which match method will be used with regex. Supported methods: match, search, findall. Default: match"
+    required: false
+    default: "match"
+  retries:
+    description: "How many retries before failure"
+    required: false
+    default: "3"
+outputs:
+  container-tags:
+    description: "List of containers tag"
+runs:
+  using: "docker"
+  image: "Dockerfile"

fetch-container-tags/main.py

@@ -0,0 +1,154 @@
+import requests
+import re
+import os
+import time
+from enum import Enum
+from dataclasses import dataclass
+
+
+class ProvidedMatchMethodNotSupportedOrIncorrect(Exception):
+    """The specified match method is not supported or incorrect"""
+    pass
+
+
+@dataclass
+class FetchRequestData:
+    package_type: str
+    package_name: str
+    organization: str
+    pat: str
+    attempts_limit: int
+
+    def get_request_url(self) -> str:
+        """
+        Description: Fetch tags for specific container
+        GH API doc: https://docs.github.com/en/rest/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-an-organization
+        """
+        url = f"https://api.github.com/orgs/{self.organization}/packages/{self.package_type}/{self.package_name}/versions"
+        return url
+
+    def get_request_headers(self) -> dict:
+        headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {self.pat}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+        return headers
+
+
+class SupportedMatchMethod(Enum):
+    MATCH = 'match'
+    SEARCH = 'search'
+    FINDALL = 'findall'
+
+
+def set_output(name, value):
+    os.system(f"""echo "{name}={value}" >> $GITHUB_OUTPUT""")
+
+
+def get_exponential_backoff_in_seconds(attempt_number: int) -> int:
+    """
+    Returns exponential back-off - depending on number of attempt.
+    Considers that `attempt_number` starts from 0.
+    Initial back-off - 4 second.
+    """
+    return pow(attempt_number + 2, 2)
+
+
+def fetch_package_metadata(request_data: FetchRequestData):
+    url: str = request_data.get_request_url()
+    headers: dict = request_data.get_request_headers()
+
+    print(f"::debug::Start fetching package metadata")
+
+    for attempt in range(request_data.attempts_limit):
+        print(
+            f"::debug::Fetching package metadata - attempt {attempt + 1} / {request_data.attempts_limit}")
+        try:
+            response = requests.get(url=url, headers=headers)
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            if attempt == request_data.attempts_limit - 1:
+                raise RuntimeError(f"{e}")
+
+            if attempt < request_data.attempts_limit - 1:
+                print(
+                    f"Exception occurred: {type(e).__name__} - {e}. Retrying.")
+                back_off = get_exponential_backoff_in_seconds(attempt)
+                print(
+                    f"::debug::Sleep for {back_off} seconds before next attempt")
+                time.sleep(back_off)
+                continue
+        break
+
+    print(f"::debug::Finish fetching metadata")
+
+    return response.json()
+
+
+def get_tags_list(package_metadata) -> list:
+    tags = []
+    for key in package_metadata:
+        tags += key.get('metadata', {}).get('container', {}).get('tags')
+    return tags
+
+
+def apply_regex_to_tags(regex: str, tags: list, perform_method: SupportedMatchMethod):
+    regex = re.compile(regex)
+    method: function = regex.match
+    if (perform_method == SupportedMatchMethod.FINDALL):
+        method = regex.match
+    if (perform_method == SupportedMatchMethod.SEARCH):
+        method = regex.search
+    print(f"::debug::Applying regex {regex} via {perform_method}")
+    return list(filter(method, tags))
+
+
+def main():
+    package_name = os.environ["INPUT_PACKAGE_NAME"]
+    package_type = "container"
+    organization = os.environ["INPUT_ORGANIZATION"]
+    pat = os.environ["INPUT_PAT"]
+    attempts_limit = int(os.environ["INPUT_RETRIES"]) + 1
+    regex = ""
+    perform_match_method_input = ""
+    perform_match_method = -1
+
+    if os.environ.get('INPUT_REGEX') is not None:
+        regex = os.environ["INPUT_REGEX"]
+
+    try:
+        perform_match_method_input = os.environ["INPUT_PERFORM_MATCH_METHOD"].upper(
+        )
+        if hasattr(SupportedMatchMethod, perform_match_method_input):
+            perform_match_method = SupportedMatchMethod[perform_match_method_input]
+        else:
+            raise ProvidedMatchMethodNotSupportedOrIncorrect(
+                f"Match method {perform_match_method_input} is not supported or incorrect")
+    except Exception as e:
+        raise RuntimeError(f"{e}")
+
+    request_data = FetchRequestData(
+        package_type=package_type,
+        package_name=package_name,
+        organization=organization,
+        pat=pat,
+        attempts_limit=attempts_limit
+    )
+
+    request_response = fetch_package_metadata(request_data)
+    container_tags = get_tags_list(request_response)
+
+    if (regex != ""):
+        container_tags = apply_regex_to_tags(
+            regex, container_tags, perform_match_method)
+
+    print("::group::Parse Semver Outputs")
+    print(f"container-tags={container_tags}")
+    print("::endgroup::")
+
+    set_output("container-tags", container_tags)
+
+
+if __name__ == "__main__":
+    main()

fetch-repo-branches/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-slim AS builder
+ADD . /app
+WORKDIR /app
+
+# We are installing a dependency here directly into our app source dir
+RUN pip install --target=/app setuptools requests
+
+# A distroless container image with Python and some basics like SSL certificates
+# https://github.com/GoogleContainerTools/distroless
+FROM gcr.io/distroless/python3-debian10
+COPY --from=builder /app /app
+WORKDIR /app
+ENV PYTHONPATH /app
+CMD ["/app/main.py"]