Merge pull request #25 from lsst/tickets/DM-32743

DM-32743: Add a container which can sync the latest schema into a registry

.github/workflows/build_sync_container.yml

@@ -0,0 +1,58 @@
+name: build_sync_container
+
+on:
+  push:
+    branches: [ main ]
+    tags: [ "*" ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build_image:
+    runs-on: ubuntu-latest
+
+    # Only do Docker builds of tagged releases and pull requests from ticket
+    # branches.  This will still trigger on pull requests from untrusted
+    # repositories whose branch names match our tickets/* branch convention,
+    # but in this case the build will fail with an error since the secret
+    # won't be set.
+    if: >
+      startsWith(github.ref, 'refs/tags/')
+      || startsWith(github.head_ref, 'tickets/')
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Define the Docker tag
+        id: vars
+        run: echo ::set-output name=tag::$(script/docker-tag.sh "$GITHUB_REF")
+
+      - name: Print the tag
+        id: print
+        run: echo ${{ steps.vars.outputs.tag }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys:
+            ${{ runner.os }}-buildx-
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: lsstdm/lsst_alert_packet:${{ steps.vars.outputs.tag }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9.7-buster AS base-image
+
+# Create a Python virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN python -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Put the latest pip and setuptools in the virtualenv
+RUN pip install --upgrade --no-cache-dir pip setuptools wheel
+
+COPY . /app
+WORKDIR /app
+
+# Install package
+RUN pip install --no-cache-dir .
+
+ENTRYPOINT ["sh", "-c"]
+CMD "syncLatestSchemaToRegistry.py --help"

python/lsst/alert/packet/bin/syncLatestSchemaToRegistry.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#
+# This file is part of alert_packet.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import argparse
+import json
+
+import fastavro
+import requests
+
+import lsst.alert.packet
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--schema-registry-url",
+        type=str,
+        default="http://alert-schemas.localhost",
+        help="URL of a Schema Registry service",
+    )
+    parser.add_argument(
+        "--subject",
+        type=str,
+        default="alert_packet",
+        help="Schema Registry subject name to use",
+    )
+    return parser.parse_args()
+
+
+def load_latest_schema():
+    schema = lsst.alert.packet.Schema.from_file()
+    normalized_schema = fastavro.schema.to_parsing_canonical_form(schema.definition)
+    return normalized_schema
+
+
+def upload_schema(registry_url, subject, normalized_schema):
+    confluent_schema = {"schema": normalized_schema}
+    payload = json.dumps(confluent_schema)
+    headers = {"Content-Type": "application/vnd.schemaregistry.v1+json"}
+    url = f"{registry_url}/subjects/{subject}/versions"
+    print(f"uploading schema to {url}")
+    response = requests.post(url=url, data=payload, headers=headers)
+    response.raise_for_status()
+    print(f"done, status={response.status_code}")
+    print(f"response text={response.text}")
+
+
+def main():
+    args = parse_args()
+    schema = load_latest_schema()
+    upload_schema(
+        args.schema_registry_url,
+        subject=args.subject,
+        normalized_schema=schema,
+    )
+
+
+if __name__ == "__main__":
+    main()

script/docker-tag.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Determine the tag for Docker images.  Takes the Git ref as its only
+# argument.
+
+set -eo pipefail
+
+if [ -n "$GITHUB_HEAD_REF" ]; then
+    # For pull requests
+    echo ${GITHUB_HEAD_REF} | sed -E 's,/,-,g'
+else
+    # For push events
+    echo ${GITHUB_REF} | sed -E 's,refs/(heads|tags)/,,' | sed -E 's,/,-,g'
+fi

setup.cfg

@@ -15,6 +15,7 @@ license = GPLv3
 install_requires =
     fastavro
     numpy
+    requests
 packages =
     lsst.alert.packet
     lsst.alert.packet.bin
@@ -33,3 +34,4 @@ lsst.alert.packet =
 console_scripts =
     validateAvroRoundTrip.py = lsst.alert.packet.bin.validateAvroRoundTrip:main
     simulateAlerts.py = lsst.alert.packet.bin.simulateAlerts:main
+    syncLatestSchemaToRegistry.py = lsst.alert.packet.bin.syncLatestSchemaToRegistry:main