Add script to generate and upload schemas

Dockerfile

@@ -4,14 +4,12 @@ MAINTAINER Frank Bertsch <frank@mozilla.com>
 # Guidelines here: https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
 ARG RUST_SPEC=stable
 ARG USER_ID="10001"
-ARG GROUP="app"
+ARG GROUP_ID="app"
 ARG HOME="/app"
 
 ENV HOME=${HOME}
-RUN mkdir ${HOME} && \
-    chown ${USER_ID}:${USER_ID} ${HOME} && \
-    groupadd --gid ${USER_ID} ${GROUP} && \
-    useradd --no-create-home --uid 10001 --gid 10001 --home-dir /app ${GROUP}
+RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
+    useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /app ${GROUP_ID}
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -20,17 +18,24 @@ RUN apt-get update && \
     apt-get clean
 
 # Install Rust and Cargo
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=${RUST_SPEC}
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=${RUST_SPEC}
 
 ENV CARGO_INSTALL_ROOT=${HOME}/.cargo
 ENV PATH ${PATH}:${HOME}/.cargo/bin
 
-# Install Google Cloud SDK
-RUN curl -sSL https://sdk.cloud.google.com | bash
-ENV PATH $PATH:$HOME/google-cloud-sdk/bin
+# Install a tagged version of jsonschema-transpiler
+RUN cargo install jsonschema-transpiler --version 1.0.0
 
+# Upgrade pip
 RUN pip install --upgrade pip
 
 WORKDIR ${HOME}
 
+ADD . ${HOME}/mozilla-schema-generator
+ENV PATH $PATH:${HOME}/mozilla-schema-generator/bin
+
+# Drop root and change ownership of the application folder to the user
+RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
 USER ${USER_ID}
+
+ENTRYPOINT ["schema_generator.sh"]

allowlist

@@ -0,0 +1,6 @@
+telemetry/core
+org-mozilla-fenix/.*
+org-mozilla-reference-browser/.*
+activity-stream/.*
+eng-workflow/bmobugs
+mobile/activation

bin/metadata_merge

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import click
+import json
+
+@click.command()
+@click.argument(
+    'metadata',
+    type=click.Path(
+        dir_okay=False,
+        file_okay=True,
+        writable=False,
+        exists=True,
+    ),
+    required=True
+)
+@click.argument(
+    'schema',
+    type=click.Path(
+        dir_okay=False,
+        file_okay=True,
+        writable=True,
+        exists=True,
+    ),
+    required=True
+)
+def main(metadata, schema):
+    print("Merging metadata {} and schema {}".format(metadata, schema))
+
+    with open(metadata, "r") as f:
+        metadata_contents = json.load(f)
+
+    with open(schema, "r") as f:
+        schema_contents = json.load(f)
+
+    properties = metadata_contents.get("properties", {})
+    required = metadata_contents.get("required", [])
+
+    if "properties" not in schema_contents:
+        schema_contents["properties"] = {}
+
+    if "required" not in schema_contents:
+        schema_contents["required"] = []
+
+    schema_contents["properties"].update(properties)
+    schema_contents["required"] += required
+
+    json_dump_args = {
+        'indent': 2,
+        'separators': (',', ': ')
+    }
+
+    with open(schema, "w") as f:
+        f.write(json.dumps(schema_contents, **json_dump_args))
+
+if __name__ == "__main__":
+    main()

bin/schema_generator.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# A script for generating schemas that are deployed into the pipeline. This
+# script handles preprocessing, filtering, and transpilation of schemas as part
+# of a pre-deployment scheme. The resulting schemas are pushed to a branch of
+# mozilla-pipeline-schemas.
+#
+# Environment variables:
+#   MPS_SSH_KEY_BASE64: A base64-encoded ssh secret key with permissions to push
+#                       to mozilla-pipeline-schemas
+#
+# Example usage:
+#   export MPS_SSH_KEY_BASE64=$(cat ~/.ssh/id_rsa | base64)
+#   make build && make run
+#
+# TODO: Update schema mapping for validation
+# TODO: Handle overwriting glean schemas
+# TODO: Include Main Ping from schema generation
+# TODO: What the heck to do with pioneer-study, a non-nested namespace
+
+set -exuo pipefail
+
+MPS_REPO_URL="git@github.com:mozilla-services/mozilla-pipeline-schemas.git"
+MPS_BRANCH_SOURCE="dev"
+MPS_BRANCH_WORKING="local-working-branch"
+MPS_BRANCH_PUBLISH="generated-schemas"
+MPS_SCHEMAS_DIR="schemas"
+
+BASE_DIR="/app"
+ALLOWLIST="$BASE_DIR/mozilla-schema-generator/allowlist"
+
+
+function setup_git_ssh() {
+    # Configure the container for pushing to github
+
+    if [[ -z "$MPS_SSH_KEY_BASE64" ]]; then
+        echo "Missing secret key" 1>&2
+        exit 1
+    fi
+
+    git config --global user.name "Generated Schema Creator"
+    git config --global user.email "dataops+pipeline-schemas@mozilla.com"
+
+    mkdir -p "$HOME/.ssh"
+
+    echo "$MPS_SSH_KEY_BASE64" | base64 --decode > /app/.ssh/id_ed25519
+    # Makes the future git-push non-interactive
+    ssh-keyscan github.com > /app/.ssh/known_hosts
+
+    chown -R "$(id -u):$(id -g)" "$HOME/.ssh"
+    chmod 700 "$HOME/.ssh"
+    chmod 700 "$HOME/.ssh/id_ed25519"
+}
+
+function setup_dependencies() {
+    # Installs mozilla-schema-generator in a virtual environment
+
+    python3 -m venv msg-venv
+    # shellcheck disable=SC1091
+    source msg-venv/bin/activate
+    pip install -e ./mozilla-schema-generator
+}
+
+function clone_and_configure_mps() {
+    # Checkout mozilla-pipeline-schemas and changes directory to prepare for
+    # schema generation.
+
+    [[ -d mozilla-pipeline-schemas ]] && rm -r mozilla-pipeline-schemas
+
+    git clone $MPS_REPO_URL
+    cd mozilla-pipeline-schemas/$MPS_SCHEMAS_DIR
+    git checkout $MPS_BRANCH_SOURCE
+    git checkout -b $MPS_BRANCH_WORKING
+}
+
+function prepare_metadata() {
+    local telemetry_metadata="metadata/telemetry-ingestion/telemetry-ingestion.1.schema.json"
+    local structured_metadata="metadata/structured-ingestion/structured-ingestion.1.schema.json"
+
+    find ./telemetry -name "*.schema.json" -type f \
+        -exec metadata_merge $telemetry_metadata {} ";"
+    find . -path ./telemetry -prune -o -name "*.schema.json" -type f \
+        -exec metadata_merge $structured_metadata {} ";"
+}
+
+function filter_schemas() {
+    # Remove metadata schemas
+    rm -rf metadata
+
+    # Pioneer-study is not nested, remove it
+    rm -rf pioneer-study
+
+    # Remove BigQuery schemas that are not in the allowlist
+    find . -name '*.bq' | grep -v -f $ALLOWLIST | xargs rm -f
+}
+
+function commit_schemas() {
+    # This method will keep a changelog of releases. If we delete and newly
+    # checkout branches everytime, that will contain a changelog of changes.
+    # Assumes the current directory is the root of the repository
+
+    find . -name "*.bq" -type f -exec git add {} +
+    git checkout ./*.schema.json
+    git commit -a -m "Interim Commit"
+
+    git checkout $MPS_BRANCH_PUBLISH || git checkout -b $MPS_BRANCH_PUBLISH
+
+    # Keep only the schemas dir
+    find . -mindepth 1 -maxdepth 1 -not -name .git -exec rm -rf {} +
+    git checkout $MPS_BRANCH_WORKING -- schemas
+    git commit -a -m "Auto-push from schema generation" || echo "Nothing to commit"
+}
+
+function main() {
+    cd $BASE_DIR
+
+    # Setup ssh key and git config
+    setup_git_ssh
+
+    # Install dependencies
+    setup_dependencies
+
+    # Pull in all schemas from MPS and change directory
+    clone_and_configure_mps
+
+    # Generate new schemas
+    mozilla-schema-generator generate-glean-ping --out-dir . --pretty
+
+    # Remove all non-json schemas (e.g. parquet)
+    find . -not -name "*.schema.json" -type f -exec rm {} +
+
+    # Add metadata to all json schemas, drop metadata schemas
+    prepare_metadata
+
+    # Add transpiled BQ schemas
+    find . -type f -name "*.schema.json" | while read -r fname; do
+        bq_out=${fname/schema.json/bq}
+        jsonschema-transpiler --type bigquery "$fname" > "$bq_out"
+    done
+
+    # Keep only allowed schemas
+    filter_schemas
+
+    # Push to branch of MPS
+    cd ../
+    commit_schemas
+    git push
+}
+
+main "$@"

docker-compose.yml

@@ -7,3 +7,5 @@ services:
       dockerfile: Dockerfile
     restart: "no"
     command: "true"
+    environment:
+      - MPS_SSH_KEY_BASE64

mozilla_schema_generator/glean_ping.py

@@ -17,7 +17,7 @@ class GleanPing(GenericPing):
     repos_url = "https://probeinfo.telemetry.mozilla.org/glean/repositories"
 
     default_probes_url = probes_url.format("glean")
-    default_pings = {"baseline", "event", "metrics"}
+    default_pings = {"baseline", "events", "metrics"}
     ignore_pings = {"default", "glean_ping_info", "glean_client_info"}
 
     def __init__(self, repo):  # TODO: Make env-url optional

tests/test_glean.py

@@ -31,7 +31,7 @@ def test_env_size(self, glean):
     def test_single_schema(self, glean, config):
         schemas = glean.generate_schema(config, split=False)
 
-        assert schemas.keys() == {"baseline", "event", "metrics"}
+        assert schemas.keys() == {"baseline", "events", "metrics"}
 
         final_schemas = {k: schemas[k][0].schema for k in schemas}
         for name, schema in final_schemas.items():