Add repeater (#423)

* add repeater model to packages

* remove test.py and update README

* update repeater readme

* add repeater model to e2e-shim and remove from e2e workflow

* add missing wildcard

* flesh out zarf package for repeater model

* update README.md

* add repeater to release workflow

---------

Co-authored-by: Gerred Dillon <gerred@defenseunicorns.com>

.github/workflows/e2e-shim.yaml

@@ -27,6 +27,9 @@ on:
       - "src/leapfrogai_ui/**"
       - "packages/ui/**"
 
+      # Catch changes to the repeater model
+      - "packages/repeater/**"
+
 
 permissions:
   contents: read

.github/workflows/e2e.yaml

@@ -31,6 +31,9 @@ on:
       - "!src/leapfrogai_ui/**"
       - "!packages/ui/**"
 
+      # Ignore changes to the repeater model
+      - "!packages/repeater/**"
+
 
 concurrency:
   group: e2e-${{ github.ref }}

.github/workflows/release.yaml

@@ -41,7 +41,7 @@ jobs:
           python-version-file: 'pyproject.toml'
 
       - name: Download Python Wheels
-        run: make setup-api-deps setup-llama-cpp-python-deps setup-vllm-deps setup-text-embeddings-deps setup-whisper-deps
+        run: make setup-api-deps setup-repeater-deps setup-llama-cpp-python-deps setup-vllm-deps setup-text-embeddings-deps setup-whisper-deps
 
       - name: Install Zarf
         uses: defenseunicorns/setup-zarf@f95763914e20e493bb5d45d63e30e17138f981d6 # v1.0.0
@@ -59,6 +59,19 @@ jobs:
           docker image prune -af
           rm zarf-package-leapfrogai-api-*.tar.zst
 
+      - name: Build and Publish repeater
+        run: |
+          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push packages/repeater
+
+          zarf package create packages/repeater --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
+          zarf package create packages/repeater --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
+
+          zarf package publish zarf-package-repeater-amd64-${{ steps.get_version.outputs.version-without-v }}.tar.zst oci://ghcr.io/defenseunicorns/packages/leapfrogai
+          zarf package publish zarf-package-repeater-arm64-${{ steps.get_version.outputs.version-without-v }}.tar.zst oci://ghcr.io/defenseunicorns/packages/leapfrogai
+
+          docker image prune -af
+          rm zarf-package-repeater-*.tar.zst
+
       - name: Build and Publish llama
         run: |
           docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push packages/llama-cpp-python

Makefile

@@ -116,8 +116,23 @@ build-whisper: local-registry setup-whisper-deps ## Build the whisper container
 	## Build the Zarf package
 	uds zarf package create packages/whisper -o packages/whisper --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm
 
+setup-repeater-deps: sdk-wheel ## Download the wheels for the optional 'repeater' dependencies
+	-rm packages/repeater/build/*.whl
+	python -m pip wheel packages/repeater -w packages/repeater/build --find-links=${SDK_DEST}
+
+build-repeater: local-registry setup-repeater-deps ## Build the repeater container and zarf package
+	## Build the image (and tag it for the local registry)
+	docker build -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} packages/repeater
+	docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION}
+
+	## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon)
+	docker push localhost:5000/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION}
+
+	## Build the Zarf package
+	uds zarf package create packages/repeater -o packages/repeater --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm
+
 build-cpu: build-api build-llama-cpp-python build-text-embeddings build-whisper ## Build all zarf packages for a cpu-enabled deployment of LFAI
 
 build-gpu: build-api build-vllm build-text-embeddings build-whisper ## Build all zarf packages for a gpu-enabled deployment of LFAI
 
-build-all: build-api build-llama-cpp-python build-vllm build-text-embeddings build-whisper ## Build all of the LFAI packages
+build-all: build-api build-llama-cpp-python build-vllm build-text-embeddings build-whisper build-repeater ## Build all of the LFAI packages

README.md

@@ -99,11 +99,11 @@ The LeapfrogAI [SDK](src/leapfrogai_sdk/) provides a standard set of protobuff a
 
 ### User Interface
 
-> GitHub Repo:
->
-> - [leapfrog-ui](https://github.com/defenseunicorns/leapfrog-ui)
+LeapfrogAI provides a [User Interface](src/leapfrogai_ui/) with support for common use-cases such as chat, summarization, and transcription.
+
+### Repeater
 
-LeapfrogAI provides some options of UI to get started with common use-cases such as chat, summarization, and transcription.
+The [repeater](packages/repeater/) "model" is a basic "backend" that parrots all inputs it receives back to the user. It is built out the same way all the actual backends are and it primarily used for testing the API.
 
 ## Usage
 
@@ -133,7 +133,7 @@ make build-all    # all of the backends
 
 **OR**
 
-You can build components individually using teh following `Make` targets:
+You can build components individually using the following `Make` targets:
 
 ```
 make build-api
@@ -183,6 +183,10 @@ python -m pip install .
 uvicorn leapfrogai_api.main:app --port 3000 --reload
 ```
 
+#### Repeater
+
+The instructions for running the basic repeater model (used for testing the API) can be found in the package [README](packages/repeater/README.md).
+
 #### Backend: llama-cpp-python
 
 To run the llama-cpp-python backend locally (starting from the root directory of the repository):

packages/repeater/Dockerfile

@@ -0,0 +1,32 @@
+ARG ARCH=amd64
+
+# hardened and slim python w/ developer tools image
+FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder
+
+WORKDIR /leapfrogai
+
+# create virtual environment for light-weight portability and minimal libraries
+RUN python3.11 -m venv .venv
+ENV PATH="/leapfrogai/.venv/bin:$PATH"
+
+# copy and install all python dependencies
+COPY build/*.whl build/
+COPY build/lfai_repeater*.whl .
+RUN pip install lfai_repeater*.whl --no-index --find-links=build/
+
+# hardened and slim python image
+FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-${ARCH}
+
+ENV PATH="/leapfrogai/.venv/bin:$PATH"
+
+WORKDIR /leapfrogai
+
+COPY --from=builder /leapfrogai/.venv/ /leapfrogai/.venv/
+
+COPY repeater.py .
+
+# Publish port
+EXPOSE 50051:50051
+
+# Run the repeater model
+ENTRYPOINT ["python", "-u", "repeater.py"]

packages/repeater/README.md

@@ -0,0 +1,36 @@
+# LeapfrogAI Repeater Backend
+
+A LeapfrogAI API-compatible repeater model that simply parrots the input it is provided back to the user. This is primarily used for quick-testing the API.
+
+
+# Usage
+
+The repeater model is used to verify that the API is able to both load configs for and send inputs to a very simple model. The repeater model fulfills this role by returning the input it recieves as output.
+
+## Local Usage
+
+Here is how to run the repeater model locally to test the API:
+
+It's easiest to set up a virtual environment to keep things clean:
+```
+python -m venv .venv
+source .venv/bin/activate
+```
+
+First install the lfai-repeater project and dependencies. From the root of the project repository:
+```
+pip install src/leapfrogai_sdk
+cd packages/repeater
+pip install .
+```
+
+Next, launch the repeater model:
+```
+python repeater.py
+```
+
+Now the basic API tests can be run in full. In a new terminal, starting from the root of the project repository:
+```
+export LFAI_RUN_REPEATER_TESTS=true    # this is needed to run the tests that require the repeater model, otherwise they get skipped
+pytest tests/pytest/test_api.py
+```

packages/repeater/chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: leapfrogai-repeater
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.6.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.4.2"

packages/repeater/chart/templates/_helpers.tpl

@@ -0,0 +1,63 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "chart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "chart.labels" -}}
+helm.sh/chart: {{ include "chart.chart" . }}
+{{ include "chart.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+app: {{ include "chart.fullname" . }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "chart.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "chart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "chart.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "chart.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

packages/repeater/chart/templates/configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.nameOverride }}-configmap"
+  namespace: {{ .Release.Namespace }}
+  labels:
+    leapfrogai: sparkle
+data:
+  models.toml: |
+    [[models]]
+        owned_by    = 'Defense Unicorns'
+        backend     = '{{ include "chart.fullname" . }}:50051'
+        type        = 'gRPC'
+        name        = '{{ .Values.nameOverride }}'

packages/repeater/chart/templates/deployment.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "chart.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+spec:
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "chart.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "chart.selectorLabels" . | nindent 8 }}
+    spec:
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

packages/repeater/chart/templates/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "chart.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    zarf.dev/connect-description: "{{ .Values.nameOverride }} gRPC endpoint."
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    zarf.dev/connect-name: "{{ .Values.nameOverride }}"
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: 50051
+      protocol: TCP
+      name: grpc
+  selector:
+    {{- include "chart.selectorLabels" . | nindent 4 }}