Initial version

.github/workflows/deploy.yml

@@ -0,0 +1,37 @@
+name: Consolidate and Deploy
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  schedule:
+    - cron: "* */1 * * *"
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install and Build
+        run: |
+          python -m pip install --upgrade poetry
+          poetry install
+          poetry run blueos_repository/consolidate.py
+          mkdir build
+          mv manifest.json build/
+
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build # The folder the action should deploy.
+        if: github.event_name != 'pull_request'

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Test Python cleanliness
+
+on:
+    pull_request:
+    push:
+
+jobs:
+  python-tests:
+    runs-on: ubuntu-latest
+
+    env:
+      python-version: 3.9 # Our base image has Python 3.9
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ env.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ env.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade poetry black isort mypy pylint aiohttp pydantic
+        poetry install
+    - name: Run tests
+      run: |
+        ./.hooks/pre-push

.hooks/pre-push

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# TOOD: add an argument for fixing errors automatically
+
+set -e
+
+echo "Running pre push hook!"
+repository_path=$(git rev-parse --show-toplevel)
+
+echo "Running isort.."
+# Run isort for each python project
+git ls-files '*.py' | xargs -I {} isort --src-path="{}" --check-only --diff "{}"
+
+echo "Running black.."
+black --check --diff "$repository_path"
+
+echo "Running pylint.."
+pylint $(git ls-files '*.py')
+
+echo "Running mypy.."
+git ls-files '*.py' | xargs --max-lines=1 mypy
+
+exit 0

README.md

@@ -1 +1,67 @@
 # BlueOS-Extensions-Repository
+
+> **Warning**
+> This is still very experimental and subject to changes!
+
+This is a repository for metadata of BlueOS Extensions.
+
+For publishing a new extension, open a pull request to this repository with the following structure:
+
+## Data in this repository
+
+/repos/yourcompany/yourextension/metadata.json
+```
+{
+    "name": "The Name of Your Extension",
+    "website": "https://your.extension.website.com/",
+    "docker": "your-dockerhub-user/your-extension-docker",
+    "description": "A brief description of your extension. This will be shown in the store card."
+}
+```
+
+/repos/yourcompany/yourextension/company_logo.png
+Your company logo
+
+/repos/yourcompany/yourextension/extension_logo.png
+Your extension logo
+
+## Data in dockerhub
+
+Additionally, we have versioned data. This data should be in each of your dockerhub tags, and use the following format:
+
+```
+LABEL version="1.0.0"
+LABEL permissions='{\
+  "ExposedPorts": {\
+    "80/tcp": {}\          // we have a server at port 80
+  },\
+  "HostConfig": {\
+    "PortBindings": {\
+      "80/tcp": [\         // our server at port 80 is automatically bound to a free port in the host
+        {\
+          "HostPort": ""\
+        }\
+      ]\
+    }\
+  }\
+}'
+LABEL authors='[\
+    {\
+        "name": "John Doe",\
+        "email": "doe@john.com"\
+    }\
+]'
+LABEL docs='http://path.to.your.docs.com'
+LABEL company='{\
+  "about": "",\
+```
+
+ - `version` is the name of the current tag, which we expect to be a valid [semver](https://semver.org/).
+ - `permissions`is a json file that follows the [Docker API payload for creating containers](https://docs.docker.com/engine/api/v1.41/#tag/Container/operation/ContainerCreate).
+ - `docs` is a url for the documentation of your extension.
+ - `authors` is a json list of authors of your extension
+ - `company` is a json, which currently only contains an "about" section for a brief description about your company.
+
+ ## How this repo works
+
+ Every time this repo changes, a Github Action runs and goes through all the .json files in here. For each of them, it reaches out to dockerhub and fetches all the available tags, extracting the metadata in LABELS and crafting a complete `manifest.json`, which is stored in this repo's gh-pages branch.

blueos_repository/consolidate.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+import asyncio
+import dataclasses
+import json
+from pathlib import Path
+from typing import Any, AsyncIterable, Dict, List, Optional, Union
+
+import aiohttp
+from registry import Registry
+
+REPO_ROOT = "https://raw.githubusercontent.com/bluerobotics/BlueOS-Extensions-Repository/master/repos"
+
+
+class EnhancedJSONEncoder(json.JSONEncoder):
+    """
+    Custom json encoder for dataclasses,
+    see https://docs.python.org/3/library/json.html#json.JSONEncoder.default
+    Returns a serializable type
+    """
+
+    def default(self, o: Any) -> Union[Any, Dict[str, Any]]:
+        if dataclasses.is_dataclass(o):
+            return dataclasses.asdict(o)
+        return super().default(o)
+
+
+@dataclasses.dataclass
+class Author:
+    name: str
+    email: str
+
+    @staticmethod
+    def from_json(json_dict: Dict[str, str]) -> "Author":
+        return Author(name=json_dict["name"], email=json_dict["email"])
+
+
+@dataclasses.dataclass
+class Company:
+    name: str
+    about: Optional[str]
+    email: Optional[str]
+
+    @staticmethod
+    def from_json(json_dict: Dict[str, str]) -> "Company":
+        return Company(name=json_dict["name"], email=json_dict.get("email", None), about=json_dict.get("about", None))
+
+
+# pylint: disable=too-many-instance-attributes
+@dataclasses.dataclass
+class Version:
+    permissions: Optional[Dict[str, Any]]
+    requirements: Optional[str]
+    tag: Optional[str]
+    website: str
+    authors: List[Author]
+    docs: Optional[str]
+    readme: Optional[str]
+    company: Optional[Company]
+    support: Optional[str]
+
+
+@dataclasses.dataclass
+class RepositoryEntry:
+    identifier: str
+    name: str
+    description: str
+    docker: str
+    versions: Dict[str, Version]
+    extension_logo: Optional[str]
+    company_logo: Optional[str]
+
+
+class Consolidator:
+    registry = Registry()
+    consolidated_data: List[RepositoryEntry] = []
+
+    @staticmethod
+    async def fetch_readme(url: str) -> str:
+        if not url.startswith("http"):
+            print(f"Invalid Readme url: {url}")
+            return "Readme not provided."
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url) as resp:
+                if resp.status != 200:
+                    print(f"Error status {resp.status}")
+                    raise Exception(f"Could not get readme {url}: status: {resp.status}")
+                if resp.content_type != "text/plain":
+                    raise Exception(f"bad response type for readme: {resp.content_type}, expected text/plain")
+                return await resp.text()
+
+    async def all_repositories(self) -> AsyncIterable[RepositoryEntry]:
+        repos = Path("./repos")
+        for repo in repos.glob("**/metadata.json"):
+            with open(repo, "r", encoding="utf-8") as individual_file:
+                company, extension_name = repo.as_posix().split("/")[1:3]
+                identifier = ".".join([company, extension_name])
+                try:
+                    data = json.load(individual_file)
+                except Exception as exc:
+                    raise Exception(f"Unable to parse file {repo}") from exc
+                company_logo = (repo / "../../company_logo.png").resolve().relative_to(repos.resolve())
+                extension_logo_file = (repo / "../extension_logo.png").resolve()
+                if extension_logo_file.exists():
+                    extension_logo = extension_logo_file.resolve().relative_to(repos.resolve())
+                else:
+                    extension_logo = company_logo
+                try:
+                    new_repo = RepositoryEntry(
+                        identifier=identifier,
+                        name=data["name"],
+                        docker=data["docker"],
+                        description=data["description"],
+                        extension_logo=f"{REPO_ROOT}/{extension_logo}" if extension_logo else None,
+                        versions={},
+                        company_logo=f"{REPO_ROOT}/{company_logo}" if company_logo else None,
+                    )
+                    yield new_repo
+                except Exception as error:
+                    raise Exception(f"unable to read file {individual_file}: {error}") from error
+
+    async def run(self) -> None:
+        async for repository in self.all_repositories():
+            for tag in await self.registry.fetch_remote_tags(repository.docker):
+                try:
+                    raw_labels = await self.registry.fetch_labels(f"{repository.docker}:{tag}")
+                    permissions = raw_labels.get("permissions", None)
+                    website = raw_labels.get("website", None)
+                    authors = raw_labels.get("authors", None)
+                    docs = raw_labels.get("docs", None)
+                    readme = raw_labels.get("readme", None)
+                    if readme is not None:
+                        readme = readme.replace(r"{tag}", tag)
+                    company_raw = raw_labels.get("company", None)
+                    company = Company.from_json(json.loads(company_raw)) if company_raw is not None else None
+                    support = raw_labels.get("support", None)
+
+                    new_version = Version(
+                        permissions=json.loads(permissions) if permissions else None,
+                        website=website,
+                        authors=json.loads(authors) if authors else [],
+                        docs=json.loads(docs) if docs else None,
+                        readme=await self.fetch_readme(readme) if readme is not None else None,
+                        company=company,
+                        support=support,
+                        requirements=raw_labels.get("requirements", None),
+                        tag=tag,
+                    )
+                    repository.versions[tag] = new_version
+                except KeyError as error:
+                    raise Exception(f"unable to parse repository {repository}: {error}") from error
+            self.consolidated_data.append(repository)
+
+        with open("manifest.json", "w", encoding="utf-8") as manifest_file:
+            manifest_file.write(json.dumps(self.consolidated_data, indent=4, cls=EnhancedJSONEncoder))
+
+
+consolidator = Consolidator()
+asyncio.run(consolidator.run())