Veilid Prototype Iteration 2

.github/workflows/cd-syft-dev.yml

@@ -121,6 +121,17 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev-${{ github.sha }}
             ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }}
 
+      - name: Build and push `grid-veilid` image to registry
+        uses: docker/build-push-action@v5
+        with:
+          context: ./packages/grid/veilid
+          file: ./packages/grid/veilid/veilid.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:${{ steps.grid.outputs.GRID_VERSION }}
+
       - name: Build Helm Chart & Copy to infra
         if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash

.github/workflows/cd-syft.yml

@@ -246,7 +246,24 @@ jobs:
           digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
           touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"
 
-      - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs
+      - name: Build and push `grid-veilid` image to DockerHub
+        id: grid-veilid-build
+        uses: docker/build-push-action@v5
+        with:
+          context: ./packages/grid/veilid
+          file: ./packages/grid/veilid/veilid.dockerfile
+          platforms: ${{ steps.release_metadata.outputs.release_platform }}
+          outputs: type=image,name=openmined/grid-veilid,push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform }}
+          cache-to: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
+
+      - name: Export digest for grid-veilid
+        run: |
+          mkdir -p /tmp/digests/grid-veilid
+          digest="${{ steps.grid-veilid-build.outputs.digest }}"
+          touch "/tmp/digests/grid-veilid/${digest#sha256:}"
+
+      - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs, grid-veilid
         uses: actions/upload-artifact@v4
         with:
           name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }}
@@ -305,6 +322,14 @@ jobs:
             -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
             $(printf 'openmined/grid-seaweedfs@sha256:%s ' *)
 
+      - name: Create manifest list and push for grid-veilid
+        working-directory: /tmp/digests/grid-veilid
+        run: |
+          docker buildx imagetools create  \
+            -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
+            -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
+            $(printf 'openmined/grid-veilid@sha256:%s ' *)
+
   deploy-syft:
     needs: [merge-docker-images]
     if: always() &&  needs.merge-docker-images.result == 'success'

.github/workflows/pr-tests-stack.yml

@@ -547,6 +547,12 @@ jobs:
         run: |
           sudo python ./scripts/patch_hosts.py --add-k3d-registry
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          large-packages: false
+
       # free 10GB of space
       - name: Remove unnecessary files
         if: matrix.os == 'ubuntu-latest'

notebooks/Testing/Veilid/Alice-Python-Server.ipynb

@@ -0,0 +1,232 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "c74990eb-d769-4117-8c88-e9210136606e",
+   "metadata": {},
+   "source": [
+    "## Alice Python Server"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20df98d8-de6c-496c-b30e-6421ac99401c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# third party\n",
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "54885cd0-f803-4911-8423-e595dc4cd7c3",
+   "metadata": {},
+   "source": [
+    "### 1. Create DHT Key and Private Route"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41d82ff3-ceda-4569-8178-8758ef635cb0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "host = \"localhost\"\n",
+    "port = 4000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d9f3cca-66a7-4e6c-a332-b38a8f5c02db",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = requests.post(f\"http://{host}:{port}/generate_dht_key\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81c6aa9d-26b4-4672-a059-643edfeeed95",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res.content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a9487e3-f5c8-468e-acd0-261e21bc3e14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = requests.get(f\"http://{host}:{port}/retrieve_dht_key\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b87e9e6-244f-47f7-a31a-fa7cbce65b88",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "self_dht_key = res.json()[\"message\"]\n",
+    "print(\"=\" * 30)\n",
+    "print(self_dht_key)\n",
+    "print(\"=\" * 30)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a8c70d99-6814-453d-80bf-d141c40ba24e",
+   "metadata": {},
+   "source": [
+    "### Send AppMessage using DHT Key to Self"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aca01ec6-1bbe-44b5-ad4a-053ba1edcfe6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me again\"}\n",
+    "app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff09ab92-3423-483a-abf3-51e8c2448cf9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "app_message.content"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4d0d9e39-bf05-4ef3-b00a-2bb605f041ee",
+   "metadata": {},
+   "source": [
+    "### Send AppCall using DHT Key to Self"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b8bc9f54-b2f0-4f88-8897-f640866ba2ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n",
+    "app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2c1c4148-461a-459e-846a-fad332a7ce3a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "app_call.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd824cca-2a7f-4ea9-9e67-1c06d1f8bec2",
+   "metadata": {},
+   "source": [
+    "### Send AppMessage using DHT Key to Peer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25cfb508-dd08-44b9-85c9-e6aa07e96a97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "peer_dht_key = input(\"Enter Peer DHT Key\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e2c1341-d840-4429-b3e5-093d8e90365e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "json_data = {\"dht_key\": peer_dht_key, \"message\": \"How are you doing , Bob\"}\n",
+    "app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "153377f6-698e-4013-9be3-0833b71ee0c4",
+   "metadata": {},
+   "source": [
+    "### Send Proxy Message "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "271d7316-eaab-438c-9192-55a4e44b9dea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = requests.get(\n",
+    "    f\"http://{host}:{port}/proxy\",\n",
+    "    json={\"url\": \"https://www.google.com\", \"method\": \"GET\", \"dht_key\": self_dht_key},\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "77e1ad1d-379a-4899-8805-c703ad437c0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res.content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73c1f0b0-d240-4964-a88b-365ea89b1bdd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}