diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b05f90042b4..f2017a5fea8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.5-beta.9 +current_version = 0.8.5-beta.10 tag = False tag_name = {new_version} commit = True diff --git a/VERSION b/VERSION index 89e9dc41ee9..7e223c385b9 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.5-beta.9" +__version__ = "0.8.5-beta.10" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/notebooks/api/0.8/06-multiple-code-requests.ipynb b/notebooks/api/0.8/06-multiple-code-requests.ipynb index 750ae7f4e8b..868cb20b91b 100644 --- a/notebooks/api/0.8/06-multiple-code-requests.ipynb +++ b/notebooks/api/0.8/06-multiple-code-requests.ipynb @@ -41,7 +41,7 @@ }, "outputs": [], "source": [ - "node = sy.orchestra.launch(name=\"test-domain-1\", port=\"auto\", dev_mode=True)" + "node = sy.orchestra.launch(name=\"test-domain-1\", port=\"auto\", reset=True, dev_mode=True)" ] }, { @@ -90,18 +90,34 @@ "cell_type": "code", "execution_count": null, "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "dataset2 = sy.Dataset(name=\"My Sample Dataset - II\")\n", + "asset2 = sy.Asset(name=\"Sample Data - II\")\n", + "asset2.set_obj(sample_data * 10)\n", + "asset2.set_mock(mock_sample_data * 10, mock_is_real=False)\n", + "asset2.set_shape(sample_data.shape)\n", + "dataset2.add_asset(asset2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", "metadata": { "tags": [] }, "outputs": [], "source": [ - "root_client.upload_dataset(dataset)" + "for dset in [dataset, dataset2]:\n", + " root_client.upload_dataset(dset)" ] }, { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "8", "metadata": { "tags": [] }, @@ -120,7 +136,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": { "tags": [] }, @@ -132,7 +148,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "10", "metadata": { "tags": [] }, @@ -144,7 +160,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -154,11 +170,11 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ - "assert len(datasets) == 1\n", + "assert len(datasets) == 2\n", "dataset_ptr = datasets[0]\n", "dataset_ptr" ] @@ -166,7 +182,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "13", "metadata": { "tags": [] }, @@ -199,7 +215,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": { "tags": [] }, @@ -216,7 +232,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": { "tags": [] }, @@ -228,7 +244,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": { "tags": [] }, @@ -242,7 +258,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": { "tags": [] }, @@ -254,7 +270,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": { "tags": [] }, @@ -267,7 +283,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "19", "metadata": { "tags": [] }, @@ -300,7 +316,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": { "tags": [] }, @@ -313,7 +329,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", "metadata": { "tags": [] }, @@ -325,7 +341,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": { "tags": [] }, @@ -338,7 +354,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -350,7 +366,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -360,7 +376,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "25", "metadata": { "tags": [] }, @@ -374,7 +390,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "26", "metadata": { "tags": [] }, @@ -386,7 +402,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "27", "metadata": { "tags": [] }, @@ -399,7 +415,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": { "tags": [] }, @@ -408,16 +424,6 @@ "assert isinstance(request_2, sy.SyftSuccess)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "ds_client.datasets" - ] - }, { "cell_type": "code", "execution_count": null, @@ -468,6 +474,38 @@ "cell_type": "code", "execution_count": null, "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "datasets = ds_client.datasets.search(name=\"My Sample Dataset - II\")\n", + "dataset_ptr2 = datasets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate if input policy is violated\n", + "sum_ptr = ds_client.code.calculate_sum(data=dataset_ptr2.assets[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(sum_ptr, sy.SyftError), sum_ptr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", "metadata": { "tags": [] }, @@ -479,7 +517,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "37", "metadata": { "tags": [] }, @@ -491,7 +529,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "38", "metadata": { "tags": [] }, @@ -503,7 +541,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "39", "metadata": { "tags": [] }, @@ -512,6 +550,14 @@ "if node.node_type.value == \"python\":\n", " node.land()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -530,7 +576,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.11.5" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb index 2ca52414c0e..b7354b469b1 100644 --- a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb +++ b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb @@ -262,7 +262,7 @@ "metadata": {}, "outputs": [], "source": [ - "@sy.syft_function_single_use(data=asset)\n", + "@sy.syft_function_single_use(df=asset)\n", "def get_mean_age(df):\n", " return df[\"Age\"].mean()" ] @@ -557,7 +557,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.4" }, "toc": { "base_numbering": 1, diff --git a/packages/.dockerignore b/packages/.dockerignore index a8628d4acb1..ba9aa4b6829 100644 --- a/packages/.dockerignore +++ b/packages/.dockerignore @@ -1,9 +1,11 @@ **/*.pyc -grid/data -grid/packer -grid/.devspace +grid/* +!grid/backend + syftcli syft/tests syft/README.md + +hagrid \ No newline at end of file diff --git a/packages/grid/VERSION b/packages/grid/VERSION index 384b842bcb3..3a6b1b83053 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.5-beta.9" +__version__ = "0.8.5-beta.10" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile index 2c859f30676..25817600671 100644 --- a/packages/grid/backend/worker_cpu.dockerfile +++ b/packages/grid/backend/worker_cpu.dockerfile @@ -9,7 +9,7 @@ # Later we'd want to uninstall old python, and then install a new python runtime... # ... but pre-built syft deps may break! -ARG SYFT_VERSION_TAG="0.8.5-beta.9" +ARG SYFT_VERSION_TAG="0.8.5-beta.10" FROM openmined/grid-backend:${SYFT_VERSION_TAG} ARG PYTHON_VERSION="3.12" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index effde818d9c..8624875b09f 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -25,7 +25,7 @@ vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io" NODE_NAME: "mynode" - VERSION: "0.8.5-beta.9" + VERSION: "0.8.5-beta.10" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible @@ -74,16 +74,11 @@ deployments: global: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} - useDefaultSecrets: true - registry: - storageSize: "5Gi" node: name: ${NODE_NAME} - rootEmail: info@openmined.org - defaultWorkerPoolCount: 1 - resourcesPreset: micro - veilid: - enabled: true + # anything that does not need devspace $env vars should go in values.dev.yaml + valuesFiles: + - ./helm/values.dev.yaml dev: mongo: diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index fac449295a7..3c6c53e5797 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.5-beta.9", + "version": "0.8.5-beta.10", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index 5d3627d58ae..1532760978d 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,9 +1,22 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.5-beta.10 + created: "2024-03-21T15:00:20.222095749Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 9cfe01e8f57eca462261a24a805b41509be2de9a0fee76e331d124ed98c4bc49 + home: https://github.com/OpenMined/PySyft/ + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.5-beta.10.tgz + version: 0.8.5-beta.10 - apiVersion: v2 appVersion: 0.8.5-beta.9 - created: "2024-03-18T06:45:00.093658129Z" + created: "2024-03-21T15:00:20.228932758Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 057f1733f2bc966e15618f62629315c8207773ef6211c79c4feb557dae15c32b @@ -16,7 +29,7 @@ entries: version: 0.8.5-beta.9 - apiVersion: v2 appVersion: 0.8.5-beta.8 - created: "2024-03-18T06:45:00.092852802Z" + created: "2024-03-21T15:00:20.228179463Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 921cbce836c3032ef62b48cc82b5b4fcbe44fb81d473cf4d69a4bf0f806eb298 @@ -29,7 +42,7 @@ entries: version: 0.8.5-beta.8 - apiVersion: v2 appVersion: 0.8.5-beta.7 - created: "2024-03-18T06:45:00.091404602Z" + created: "2024-03-21T15:00:20.227422221Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 75482e955b2b9853a80bd653afb1d56535f78f3bfb7726798522307eb3effbbd @@ -42,7 +55,7 @@ entries: version: 0.8.5-beta.7 - apiVersion: v2 appVersion: 0.8.5-beta.6 - created: "2024-03-18T06:45:00.09065511Z" + created: "2024-03-21T15:00:20.226000452Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6a2dfaf65ca855e1b3d7b966d4ff291e6fcbe761e2fc2a78033211ccd3a75de0 @@ -55,7 +68,7 @@ entries: version: 0.8.5-beta.6 - apiVersion: v2 appVersion: 0.8.5-beta.5 - created: "2024-03-18T06:45:00.089881312Z" + created: "2024-03-21T15:00:20.22516844Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: fead03823bef04d66901d563aa755c68ab277f72b126aaa6f0dce76a6f3bdb6d @@ -68,7 +81,7 @@ entries: version: 0.8.5-beta.5 - apiVersion: v2 appVersion: 0.8.5-beta.4 - created: "2024-03-18T06:45:00.089125819Z" + created: "2024-03-21T15:00:20.224413643Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 93e4539d5726a7fd0d6a3e93d1c17c6a358a923ddc01d102eab22f37377502ab @@ -81,7 +94,7 @@ entries: version: 0.8.5-beta.4 - apiVersion: v2 appVersion: 0.8.5-beta.3 - created: "2024-03-18T06:45:00.088342233Z" + created: "2024-03-21T15:00:20.223612789Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: f91e9390edf3441469048f5da646099af98f8b6d199409d0e2c1e6da3a51f054 @@ -94,7 +107,7 @@ entries: version: 0.8.5-beta.3 - apiVersion: v2 appVersion: 0.8.5-beta.2 - created: "2024-03-18T06:45:00.087573506Z" + created: "2024-03-21T15:00:20.22286839Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 59159c3aa4888038edc3c0135c83402363d7a0639fe62966a1e9d4928a364fa8 @@ -107,7 +120,7 @@ entries: version: 0.8.5-beta.2 - apiVersion: v2 appVersion: 0.8.5-beta.1 - created: "2024-03-18T06:45:00.086811641Z" + created: "2024-03-21T15:00:20.221280499Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 65aeb74c52ed8ba5474af500b4c1188a570ee4cb1f2a2da356b3488d28356ed9 @@ -119,7 +132,7 @@ entries: version: 0.8.5-beta.1 - apiVersion: v2 appVersion: 0.8.4 - created: "2024-03-18T06:45:00.086392947Z" + created: "2024-03-21T15:00:20.220902524Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 08afea8e3a9eef225b7e611f0bc1216c140053ef8e51439b02337faeac621fd0 @@ -131,7 +144,7 @@ entries: version: 0.8.4 - apiVersion: v2 appVersion: 0.8.4-beta.31 - created: "2024-03-18T06:45:00.083232335Z" + created: "2024-03-21T15:00:20.217749079Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: fabf3e2f37e53fa623f5d3d99b00feae06e278e5cd63bce419089946312ab1fc @@ -143,7 +156,7 @@ entries: version: 0.8.4-beta.31 - apiVersion: v2 appVersion: 0.8.4-beta.30 - created: "2024-03-18T06:45:00.082831405Z" + created: "2024-03-21T15:00:20.217313116Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6e8f792709f73ec14eab48a268bdf50a4505b340bd142cddd7c7bfffd94009ad @@ -155,7 +168,7 @@ entries: version: 0.8.4-beta.30 - apiVersion: v2 appVersion: 0.8.4-beta.29 - created: "2024-03-18T06:45:00.082010139Z" + created: "2024-03-21T15:00:20.216572915Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 4c985d6a9b3456769c4013f9e85e7374c0f963d2d27627e61f914f5537de1971 @@ -167,7 +180,7 @@ entries: version: 0.8.4-beta.29 - apiVersion: v2 appVersion: 0.8.4-beta.28 - created: "2024-03-18T06:45:00.081610241Z" + created: "2024-03-21T15:00:20.216175785Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: bd2aa3c92c768c47c502e31a326f341addcb34e64d22cdcbf5cc3f19689d859c @@ -179,7 +192,7 @@ entries: version: 0.8.4-beta.28 - apiVersion: v2 appVersion: 0.8.4-beta.27 - created: "2024-03-18T06:45:00.081212608Z" + created: "2024-03-21T15:00:20.215773945Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: e8ad0869993af39d7adda8cb868dc0b24cfb63b4bb9820dc579939c1007a60ba @@ -191,7 +204,7 @@ entries: version: 0.8.4-beta.27 - apiVersion: v2 appVersion: 0.8.4-beta.26 - created: "2024-03-18T06:45:00.080810375Z" + created: "2024-03-21T15:00:20.215370693Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 30dccf630aa25a86a03c67572fe5411687d8ce6d58def448ea10efdba2b85e3a @@ -203,7 +216,7 @@ entries: version: 0.8.4-beta.26 - apiVersion: v2 appVersion: 0.8.4-beta.25 - created: "2024-03-18T06:45:00.080402933Z" + created: "2024-03-21T15:00:20.214958544Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b6e2043bcf5a0335967d770c7939f5a7832955359a7d871c90b265660ff26e5f @@ -215,7 +228,7 @@ entries: version: 0.8.4-beta.25 - apiVersion: v2 appVersion: 0.8.4-beta.24 - created: "2024-03-18T06:45:00.079991824Z" + created: "2024-03-21T15:00:20.214533872Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b19efa95394d50bb8d76da6ec306de5d3bb9ea55371fafea95a1282a697fa33e @@ -227,7 +240,7 @@ entries: version: 0.8.4-beta.24 - apiVersion: v2 appVersion: 0.8.4-beta.23 - created: "2024-03-18T06:45:00.079559104Z" + created: "2024-03-21T15:00:20.214110573Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 5c5d05c15bff548574896118ce92335ae10c5b78f5307fe9b2618e5a5aa71a5c @@ -239,7 +252,7 @@ entries: version: 0.8.4-beta.23 - apiVersion: v2 appVersion: 0.8.4-beta.22 - created: "2024-03-18T06:45:00.078779838Z" + created: "2024-03-21T15:00:20.213604048Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0160dbce938198132ca9cd2a5cb362816344687291f5b6d7cf6de8f2855e9414 @@ -251,7 +264,7 @@ entries: version: 0.8.4-beta.22 - apiVersion: v2 appVersion: 0.8.4-beta.21 - created: "2024-03-18T06:45:00.077924749Z" + created: "2024-03-21T15:00:20.212845636Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7dce153d2fcae7513e9c132e139b2721fd975ea3cc43a370e34dbeb2a1b7f683 @@ -263,7 +276,7 @@ entries: version: 0.8.4-beta.21 - apiVersion: v2 appVersion: 0.8.4-beta.20 - created: "2024-03-18T06:45:00.077516425Z" + created: "2024-03-21T15:00:20.211987616Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c51189a187bbf24135382e25cb00964e0330dfcd3b2f0c884581a6686f05dd28 @@ -275,7 +288,7 @@ entries: version: 0.8.4-beta.20 - apiVersion: v2 appVersion: 0.8.4-beta.19 - created: "2024-03-18T06:45:00.076539929Z" + created: "2024-03-21T15:00:20.211025982Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 8219575dedb42fa2ddbf2768a4e9afbfacbc2dff7e953d77c7b10a41b78dc687 @@ -287,7 +300,7 @@ entries: version: 0.8.4-beta.19 - apiVersion: v2 appVersion: 0.8.4-beta.18 - created: "2024-03-18T06:45:00.076146062Z" + created: "2024-03-21T15:00:20.210629462Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6418cde559cf12f1f7fea5a2b123bba950e50eeb3be002441827d2ab7f9e4ef7 @@ -299,7 +312,7 @@ entries: version: 0.8.4-beta.18 - apiVersion: v2 appVersion: 0.8.4-beta.17 - created: "2024-03-18T06:45:00.075743018Z" + created: "2024-03-21T15:00:20.21023145Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 71b39c5a4c64037eadbb154f7029282ba90d9a0d703f8d4c7dfc1ba2f5d81498 @@ -311,7 +324,7 @@ entries: version: 0.8.4-beta.17 - apiVersion: v2 appVersion: 0.8.4-beta.16 - created: "2024-03-18T06:45:00.075308245Z" + created: "2024-03-21T15:00:20.209822517Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9c9840a7c9476dbb08e0ac83926330718fe50c89879752dd8f92712b036109c0 @@ -323,7 +336,7 @@ entries: version: 0.8.4-beta.16 - apiVersion: v2 appVersion: 0.8.4-beta.15 - created: "2024-03-18T06:45:00.074635476Z" + created: "2024-03-21T15:00:20.209371426Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0955fd22da028315e30c68132cbfa4bdc82bae622039bcfce0de339707bb82eb @@ -335,7 +348,7 @@ entries: version: 0.8.4-beta.15 - apiVersion: v2 appVersion: 0.8.4-beta.14 - created: "2024-03-18T06:45:00.073825221Z" + created: "2024-03-21T15:00:20.208975868Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 56208571956abe20ed7a5cc1867cab2667ed792c63e53d0e8bb70a9b438b7bf6 @@ -347,7 +360,7 @@ entries: version: 0.8.4-beta.14 - apiVersion: v2 appVersion: 0.8.4-beta.13 - created: "2024-03-18T06:45:00.073159866Z" + created: "2024-03-21T15:00:20.208629121Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: d7222c72412b6ee5833fbb07d2549be179cdfc7ccd89e0ad947d112fce799b83 @@ -359,7 +372,7 @@ entries: version: 0.8.4-beta.13 - apiVersion: v2 appVersion: 0.8.4-beta.12 - created: "2024-03-18T06:45:00.072485514Z" + created: "2024-03-21T15:00:20.208282234Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: af08c723756e397962b2d5190dedfd50797b771c5caf58b93a6f65d8fa24785c @@ -371,7 +384,7 @@ entries: version: 0.8.4-beta.12 - apiVersion: v2 appVersion: 0.8.4-beta.11 - created: "2024-03-18T06:45:00.071834006Z" + created: "2024-03-21T15:00:20.2079366Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a0235835ba57d185a83dd8a26281fa37b2077c3a37fe3a1c50585005695927e3 @@ -383,7 +396,7 @@ entries: version: 0.8.4-beta.11 - apiVersion: v2 appVersion: 0.8.4-beta.10 - created: "2024-03-18T06:45:00.07089251Z" + created: "2024-03-21T15:00:20.207586717Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 910ddfeba0c5e66651500dd11404afff092adc0f768ed68e0d93b04b83aa4388 @@ -395,7 +408,7 @@ entries: version: 0.8.4-beta.10 - apiVersion: v2 appVersion: 0.8.4-beta.9 - created: "2024-03-18T06:45:00.085950099Z" + created: "2024-03-21T15:00:20.220485005Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c25ca8a9f072d6a5d02232448deaef5668aca05f24dfffbba3ebe30a4f75bb26 @@ -407,7 +420,7 @@ entries: version: 0.8.4-beta.9 - apiVersion: v2 appVersion: 0.8.4-beta.8 - created: "2024-03-18T06:45:00.085188244Z" + created: "2024-03-21T15:00:20.220102872Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7249a39d4137e457b369384ba0a365c271c780d93a8327ce25083df763c39999 @@ -419,7 +432,7 @@ entries: version: 0.8.4-beta.8 - apiVersion: v2 appVersion: 0.8.4-beta.7 - created: "2024-03-18T06:45:00.084561812Z" + created: "2024-03-21T15:00:20.219508581Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ee750c7c8d6ea05bd447375e624fdd7f66dd87680ab81f7b7e73df7379a9024a @@ -431,7 +444,7 @@ entries: version: 0.8.4-beta.7 - apiVersion: v2 appVersion: 0.8.4-beta.6 - created: "2024-03-18T06:45:00.084235302Z" + created: "2024-03-21T15:00:20.218752641Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0e046be9f73df7444a995608c59af16fab9030b139b2acb4d6db6185b8eb5337 @@ -443,7 +456,7 @@ entries: version: 0.8.4-beta.6 - apiVersion: v2 appVersion: 0.8.4-beta.5 - created: "2024-03-18T06:45:00.083907057Z" + created: "2024-03-21T15:00:20.218417015Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b56e9a23d46810eccdb4cf5272cc05126da3f6db314e541959c3efb5f260620b @@ -455,7 +468,7 @@ entries: version: 0.8.4-beta.5 - apiVersion: v2 appVersion: 0.8.4-beta.4 - created: "2024-03-18T06:45:00.083578363Z" + created: "2024-03-21T15:00:20.218084014Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1d5808ecaf55391f3b27ae6236400066508acbd242e33db24a1ab4bffa77409e @@ -467,7 +480,7 @@ entries: version: 0.8.4-beta.4 - apiVersion: v2 appVersion: 0.8.4-beta.3 - created: "2024-03-18T06:45:00.08239541Z" + created: "2024-03-21T15:00:20.216910976Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b64efa8529d82be56c6ab60487ed24420a5614d96d2509c1f93c1003eda71a54 @@ -479,7 +492,7 @@ entries: version: 0.8.4-beta.3 - apiVersion: v2 appVersion: 0.8.4-beta.2 - created: "2024-03-18T06:45:00.07710195Z" + created: "2024-03-21T15:00:20.211575778Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -495,7 +508,7 @@ entries: version: 0.8.4-beta.2 - apiVersion: v2 appVersion: 0.8.4-beta.1 - created: "2024-03-18T06:45:00.070107121Z" + created: "2024-03-21T15:00:20.207179648Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -511,7 +524,7 @@ entries: version: 0.8.4-beta.1 - apiVersion: v2 appVersion: 0.8.3 - created: "2024-03-18T06:45:00.069563974Z" + created: "2024-03-21T15:00:20.206021034Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -527,7 +540,7 @@ entries: version: 0.8.3 - apiVersion: v2 appVersion: 0.8.3-beta.6 - created: "2024-03-18T06:45:00.068891066Z" + created: "2024-03-21T15:00:20.205337429Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -543,7 +556,7 @@ entries: version: 0.8.3-beta.6 - apiVersion: v2 appVersion: 0.8.3-beta.5 - created: "2024-03-18T06:45:00.068282147Z" + created: "2024-03-21T15:00:20.204768618Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -559,7 +572,7 @@ entries: version: 0.8.3-beta.5 - apiVersion: v2 appVersion: 0.8.3-beta.4 - created: "2024-03-18T06:45:00.067703124Z" + created: "2024-03-21T15:00:20.204198835Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -575,7 +588,7 @@ entries: version: 0.8.3-beta.4 - apiVersion: v2 appVersion: 0.8.3-beta.3 - created: "2024-03-18T06:45:00.067043941Z" + created: "2024-03-21T15:00:20.203497357Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -591,7 +604,7 @@ entries: version: 0.8.3-beta.3 - apiVersion: v2 appVersion: 0.8.3-beta.2 - created: "2024-03-18T06:45:00.066452575Z" + created: "2024-03-21T15:00:20.202958973Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -607,7 +620,7 @@ entries: version: 0.8.3-beta.2 - apiVersion: v2 appVersion: 0.8.3-beta.1 - created: "2024-03-18T06:45:00.065887718Z" + created: "2024-03-21T15:00:20.202400802Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -623,7 +636,7 @@ entries: version: 0.8.3-beta.1 - apiVersion: v2 appVersion: 0.8.2 - created: "2024-03-18T06:45:00.065045453Z" + created: "2024-03-21T15:00:20.201831791Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -639,7 +652,7 @@ entries: version: 0.8.2 - apiVersion: v2 appVersion: 0.8.2-beta.60 - created: "2024-03-18T06:45:00.06388954Z" + created: "2024-03-21T15:00:20.200428085Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -655,7 +668,7 @@ entries: version: 0.8.2-beta.60 - apiVersion: v2 appVersion: 0.8.2-beta.59 - created: "2024-03-18T06:45:00.063260423Z" + created: "2024-03-21T15:00:20.199753116Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -671,7 +684,7 @@ entries: version: 0.8.2-beta.59 - apiVersion: v2 appVersion: 0.8.2-beta.58 - created: "2024-03-18T06:45:00.062614505Z" + created: "2024-03-21T15:00:20.199123161Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -687,7 +700,7 @@ entries: version: 0.8.2-beta.58 - apiVersion: v2 appVersion: 0.8.2-beta.57 - created: "2024-03-18T06:45:00.061942157Z" + created: "2024-03-21T15:00:20.198472026Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -703,7 +716,7 @@ entries: version: 0.8.2-beta.57 - apiVersion: v2 appVersion: 0.8.2-beta.56 - created: "2024-03-18T06:45:00.061312359Z" + created: "2024-03-21T15:00:20.197838295Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -719,7 +732,7 @@ entries: version: 0.8.2-beta.56 - apiVersion: v2 appVersion: 0.8.2-beta.53 - created: "2024-03-18T06:45:00.06067629Z" + created: "2024-03-21T15:00:20.197165149Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -735,7 +748,7 @@ entries: version: 0.8.2-beta.53 - apiVersion: v2 appVersion: 0.8.2-beta.52 - created: "2024-03-18T06:45:00.060027977Z" + created: "2024-03-21T15:00:20.196519956Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -751,7 +764,7 @@ entries: version: 0.8.2-beta.52 - apiVersion: v2 appVersion: 0.8.2-beta.51 - created: "2024-03-18T06:45:00.059062379Z" + created: "2024-03-21T15:00:20.195855266Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -767,7 +780,7 @@ entries: version: 0.8.2-beta.51 - apiVersion: v2 appVersion: 0.8.2-beta.50 - created: "2024-03-18T06:45:00.058128702Z" + created: "2024-03-21T15:00:20.194420587Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -783,7 +796,7 @@ entries: version: 0.8.2-beta.50 - apiVersion: v2 appVersion: 0.8.2-beta.49 - created: "2024-03-18T06:45:00.057499185Z" + created: "2024-03-21T15:00:20.193778318Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -799,7 +812,7 @@ entries: version: 0.8.2-beta.49 - apiVersion: v2 appVersion: 0.8.2-beta.48 - created: "2024-03-18T06:45:00.056859278Z" + created: "2024-03-21T15:00:20.193098581Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -815,7 +828,7 @@ entries: version: 0.8.2-beta.48 - apiVersion: v2 appVersion: 0.8.2-beta.47 - created: "2024-03-18T06:45:00.056223188Z" + created: "2024-03-21T15:00:20.192455351Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -831,7 +844,7 @@ entries: version: 0.8.2-beta.47 - apiVersion: v2 appVersion: 0.8.2-beta.46 - created: "2024-03-18T06:45:00.05542239Z" + created: "2024-03-21T15:00:20.191899474Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -847,7 +860,7 @@ entries: version: 0.8.2-beta.46 - apiVersion: v2 appVersion: 0.8.2-beta.45 - created: "2024-03-18T06:45:00.05482359Z" + created: "2024-03-21T15:00:20.191335162Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -863,7 +876,7 @@ entries: version: 0.8.2-beta.45 - apiVersion: v2 appVersion: 0.8.2-beta.44 - created: "2024-03-18T06:45:00.05420856Z" + created: "2024-03-21T15:00:20.190762945Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -879,7 +892,7 @@ entries: version: 0.8.2-beta.44 - apiVersion: v2 appVersion: 0.8.2-beta.43 - created: "2024-03-18T06:45:00.053526184Z" + created: "2024-03-21T15:00:20.190159349Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -895,7 +908,7 @@ entries: version: 0.8.2-beta.43 - apiVersion: v2 appVersion: 0.8.2-beta.41 - created: "2024-03-18T06:45:00.05216895Z" + created: "2024-03-21T15:00:20.188907875Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -911,7 +924,7 @@ entries: version: 0.8.2-beta.41 - apiVersion: v2 appVersion: 0.8.2-beta.40 - created: "2024-03-18T06:45:00.051517321Z" + created: "2024-03-21T15:00:20.187856834Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -927,7 +940,7 @@ entries: version: 0.8.2-beta.40 - apiVersion: v2 appVersion: 0.8.2-beta.39 - created: "2024-03-18T06:45:00.050946964Z" + created: "2024-03-21T15:00:20.187315916Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -943,7 +956,7 @@ entries: version: 0.8.2-beta.39 - apiVersion: v2 appVersion: 0.8.2-beta.38 - created: "2024-03-18T06:45:00.050355949Z" + created: "2024-03-21T15:00:20.186759839Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -959,7 +972,7 @@ entries: version: 0.8.2-beta.38 - apiVersion: v2 appVersion: 0.8.2-beta.37 - created: "2024-03-18T06:45:00.049783638Z" + created: "2024-03-21T15:00:20.186183564Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -975,7 +988,7 @@ entries: version: 0.8.2-beta.37 - apiVersion: v2 appVersion: 0.8.1 - created: "2024-03-18T06:45:00.049190198Z" + created: "2024-03-21T15:00:20.185566663Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -989,4 +1002,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1.tgz version: 0.8.1 -generated: "2024-03-18T06:45:00.048477024Z" +generated: "2024-03-21T15:00:20.184822705Z" diff --git a/packages/grid/helm/repo/syft-0.8.5-beta.10.tgz b/packages/grid/helm/repo/syft-0.8.5-beta.10.tgz new file mode 100644 index 00000000000..28a2949e230 Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.5-beta.10.tgz differ diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index dc28fcd8abe..88753c6c4e1 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: syft description: Perform numpy-like analysis on data that remains in someone elses server type: application -version: "0.8.5-beta.9" -appVersion: "0.8.5-beta.9" +version: "0.8.5-beta.10" +appVersion: "0.8.5-beta.10" home: https://github.com/OpenMined/PySyft/ icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png diff --git a/packages/grid/helm/syft/templates/NOTES.txt b/packages/grid/helm/syft/templates/NOTES.txt index 3f985bcdcfe..a1eec6fe2f0 100644 --- a/packages/grid/helm/syft/templates/NOTES.txt +++ b/packages/grid/helm/syft/templates/NOTES.txt @@ -71,7 +71,7 @@ "ExecutionOutput": { "1": { "version": 1, - "hash": "abb4ce9172fbba0ea03fcbea7addb06176c8dba6dbcb7143cde350617528a5b7", + "hash": "201c8abcb6595a64140ad0c3b058557229c7790a25fb55ed229ae0efcb63ad07", "action": "add" } }, diff --git a/packages/grid/helm/syft/templates/_secrets.tpl b/packages/grid/helm/syft/templates/_secrets.tpl index 4d0ad6bd153..8a7d57f3bb8 100644 --- a/packages/grid/helm/syft/templates/_secrets.tpl +++ b/packages/grid/helm/syft/templates/_secrets.tpl @@ -24,17 +24,24 @@ Params: {{- end -}} {{/* -Re-use or set a new randomly generated secret value from an existing secret. -If global.useDefaultSecrets is set to true, the default value will be used if the secret does not exist. +Set a value for a Secret. +- If the secret exists, the existing value will be re-used. +- If "randomDefault"=true, a random value will be generated. +- If "randomDefault"=false, the "default" value will be used. Usage: - {{- include "common.secrets.set " (dict "secret" "some-secret-name" "default" "default-value" "context" $ ) }} + Generate random secret of length 64 + {{- include "common.secrets.set " (dict "secret" "some-secret-name" "randomDefault" true "randomLength" 64 "context" $ ) }} + + Use a static default value (with random disabled) + {{- include "common.secrets.set " (dict "secret" "some-secret-name" "default" "default-value" "randomDefault" false "context" $ ) }} Params: secret - String (Required) - Name of the 'Secret' resource where the key is stored. key - String - (Required) - Name of the key in the secret. - default - String - (Optional) - Default value to use if the secret does not exist. - length - Int - (Optional) - The length of the generated secret. Default is 32. + randomDefault - Bool - (Optional) - If true, a random value will be generated if secret does note exit. + randomLength - Int - (Optional) - The length of the generated secret. Default is 32. + default - String - (Optional) - Default value to use if the secret does not exist if "randomDefault" is set to false. context - Context (Required) - Parent context. */}} {{- define "common.secrets.set" -}} @@ -43,11 +50,11 @@ Params: {{- if $existingSecret -}} {{- $secretVal = $existingSecret -}} - {{- else if .context.Values.global.useDefaultSecrets -}} - {{- $secretVal = .default | b64enc -}} - {{- else -}} - {{- $length := .length | default 32 -}} + {{- else if .randomDefault -}} + {{- $length := .randomLength | default 32 -}} {{- $secretVal = randAlphaNum $length | b64enc -}} + {{- else -}} + {{- $secretVal = .default | required (printf "default value required for secret=%s key=%s" .secret .key) |b64enc -}} {{- end -}} {{- printf "%s" $secretVal -}} diff --git a/packages/grid/helm/syft/templates/backend/backend-secret.yaml b/packages/grid/helm/syft/templates/backend/backend-secret.yaml index 12b14be20bd..1aec7d9bbc9 100644 --- a/packages/grid/helm/syft/templates/backend/backend-secret.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-secret.yaml @@ -11,6 +11,7 @@ data: defaultRootPassword: {{ include "common.secrets.set" (dict "secret" $secretName "key" "defaultRootPassword" - "default" .Values.node.defaultSecret.defaultRootPassword + "randomDefault" .Values.global.randomizedSecrets + "default" .Values.node.secret.defaultRootPassword "context" $) }} diff --git a/packages/grid/helm/syft/templates/mongo/mongo-secret.yaml b/packages/grid/helm/syft/templates/mongo/mongo-secret.yaml index a58fb2b72c6..02c58d276ca 100644 --- a/packages/grid/helm/syft/templates/mongo/mongo-secret.yaml +++ b/packages/grid/helm/syft/templates/mongo/mongo-secret.yaml @@ -11,6 +11,7 @@ data: rootPassword: {{ include "common.secrets.set" (dict "secret" $secretName "key" "rootPassword" - "default" .Values.mongo.defaultSecret.rootPassword + "randomDefault" .Values.global.randomizedSecrets + "default" .Values.mongo.secret.rootPassword "context" $) }} diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml index c4a0e9b5b09..b0183765115 100644 --- a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml @@ -12,7 +12,8 @@ data: s3RootPassword: {{ include "common.secrets.set" (dict "secret" $secretName "key" "s3RootPassword" - "default" .Values.seaweedfs.defaultSecret.s3RootPassword + "randomDefault" .Values.global.randomizedSecrets + "default" .Values.seaweedfs.secret.s3RootPassword "context" $) }} {{ end }} diff --git a/packages/grid/helm/syft/templates/veilid/veilid-deployment.yaml b/packages/grid/helm/syft/templates/veilid/veilid-deployment.yaml index 1b05569837a..58aef67597a 100644 --- a/packages/grid/helm/syft/templates/veilid/veilid-deployment.yaml +++ b/packages/grid/helm/syft/templates/veilid/veilid-deployment.yaml @@ -27,12 +27,14 @@ spec: resources: {{ include "common.resources.set" (dict "resources" .Values.veilid.resources "preset" .Values.veilid.resourcesPreset) | nindent 12 }} env: - - name: VEILID_FLAGS - value: {{ .Values.veilid.serverFlags | quote }} - name: UVICORN_LOG_LEVEL value: {{ .Values.veilid.uvicornLogLevel }} - name: APP_LOG_LEVEL value: {{ .Values.veilid.appLogLevel }} + {{- if .Values.veilid.serverFlags }} + - name: VEILID_FLAGS + value: {{ .Values.veilid.serverFlags | quote }} + {{- end }} {{- if .Values.veilid.env }} {{- toYaml .Values.veilid.env | nindent 12 }} {{- end }} diff --git a/packages/grid/helm/syft/templates/veilid/veilid-service.yaml b/packages/grid/helm/syft/templates/veilid/veilid-service.yaml index 4b71381b9cc..dc2beb5ec99 100644 --- a/packages/grid/helm/syft/templates/veilid/veilid-service.yaml +++ b/packages/grid/helm/syft/templates/veilid/veilid-service.yaml @@ -13,7 +13,7 @@ spec: app.kubernetes.io/component: veilid ports: - name: python-server - port: 80 protocol: TCP + port: 80 targetPort: 4000 -{{ end }} \ No newline at end of file +{{ end }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 2b64a8998d1..7625c9be08e 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -1,10 +1,10 @@ global: # Affects only backend, frontend, and seaweedfs containers registry: docker.io - version: 0.8.5-beta.9 + version: 0.8.5-beta.10 - # Force default secret values for development. DO NOT USE IN PRODUCTION - useDefaultSecrets: false + # Force default secret values for development. DO NOT SET THIS TO FALSE IN PRODUCTION + randomizedSecrets: true mongo: # MongoDB config @@ -24,9 +24,9 @@ mongo: # Mongo secret name. Override this if you want to use a self-managed secret. secretKeyName: mongo-secret - # Dev mode default passwords - defaultSecret: - rootPassword: example + # custom secret values + secret: + rootPassword: null frontend: # Extra environment vars @@ -62,9 +62,9 @@ seaweedfs: resourcesPreset: nano resources: null - # Dev mode default passwords - defaultSecret: - s3RootPassword: admin + # custom secret values + secret: + s3RootPassword: null proxy: # Extra environment vars @@ -122,9 +122,9 @@ node: # - defaultRootPassword secretKeyName: backend-secret - # Dev mode default passwords - defaultSecret: - defaultRootPassword: changethis + # custom secret values + secret: + defaultRootPassword: null ingress: hostname: null # do not make this localhost @@ -152,7 +152,7 @@ ingress: # ---------------------------------------- veilid: enabled: false - serverFlags: "" + serverFlags: null appLogLevel: "info" uvicornLogLevel: "info" @@ -161,4 +161,4 @@ veilid: # Pod Resource Limits resourcesPreset: nano - resources: null \ No newline at end of file + resources: null diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml new file mode 100644 index 00000000000..62e4d16b234 --- /dev/null +++ b/packages/grid/helm/values.dev.yaml @@ -0,0 +1,26 @@ +# Helm chart values used for development and testing +# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` + +global: + randomizedSecrets: false + +registry: + storageSize: "5Gi" + +node: + rootEmail: info@openmined.org + defaultWorkerPoolCount: 1 + + secret: + defaultRootPassword: changethis + +mongo: + secret: + rootPassword: example + +seaweedfs: + secret: + s3RootPassword: admin + +veilid: + enabled: true diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 465bcf58499..0d1612eec58 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -31,7 +31,7 @@ data: RABBITMQ_VERSION: 3 SEAWEEDFS_VERSION: 3.59 DOCKER_IMAGE_SEAWEEDFS: chrislusf/seaweedfs:3.55 - VERSION: 0.8.5-beta.9 + VERSION: 0.8.5-beta.10 VERSION_HASH: unknown STACK_API_KEY: "" diff --git a/packages/grid/podman/podman-kube/podman-syft-kube.yaml b/packages/grid/podman/podman-kube/podman-syft-kube.yaml index 394724ed21a..418eaca3bc2 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube.yaml @@ -41,7 +41,7 @@ spec: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-backend:0.8.5-beta.9 + image: docker.io/openmined/grid-backend:0.8.5-beta.10 imagePullPolicy: IfNotPresent resources: {} tty: true @@ -57,7 +57,7 @@ spec: envFrom: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-frontend:0.8.5-beta.9 + image: docker.io/openmined/grid-frontend:0.8.5-beta.10 imagePullPolicy: IfNotPresent resources: {} tty: true diff --git a/packages/grid/veilid/.dockerignore b/packages/grid/veilid/.dockerignore new file mode 100644 index 00000000000..cc8cc888f79 --- /dev/null +++ b/packages/grid/veilid/.dockerignore @@ -0,0 +1 @@ +veilid \ No newline at end of file diff --git a/packages/grid/veilid/start.sh b/packages/grid/veilid/start.sh index 86572d98e66..0675243d3d4 100644 --- a/packages/grid/veilid/start.sh +++ b/packages/grid/veilid/start.sh @@ -16,6 +16,6 @@ then RELOAD="--reload" fi -/veilid/veilid-server -c /veilid/veilid-server.conf $VEILID_FLAGS & +/veilid/veilid-server -c /veilid/veilid-server.conf $VEILID_FLAGS & exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $UVICORN_LOG_LEVEL "$APP_MODULE" \ No newline at end of file diff --git a/packages/hagrid/hagrid/deps.py b/packages/hagrid/hagrid/deps.py index f650f957abb..3b6a1a657ad 100644 --- a/packages/hagrid/hagrid/deps.py +++ b/packages/hagrid/hagrid/deps.py @@ -36,7 +36,7 @@ from .nb_output import NBOutput from .version import __version__ -LATEST_BETA_SYFT = "0.8.5-beta.9" +LATEST_BETA_SYFT = "0.8.5-beta.10" DOCKER_ERROR = """ You are running an old version of docker, possibly on Linux. You need to install v2. diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 61082ac533c..a6facca1628 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 hagrid_version: 0.3.112 -syft_version: 0.8.5-beta.9 -dockerTag: 0.8.5-beta.9 +syft_version: 0.8.5-beta.10 +dockerTag: 0.8.5-beta.10 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: 32e0f11a572d30d88acf5061daf7c2927ef38d7f +hash: 95c17b2d0d4d2ab97727315eb1545b3fd74f8fdc target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 2440172d448..8ab47ba1c43 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = syft -version = attr: "0.8.5-beta.9" +version = attr: "0.8.5-beta.10" description = Perform numpy-like analysis on data that remains in someone elses server author = OpenMined author_email = info@openmined.org diff --git a/packages/syft/src/syft/VERSION b/packages/syft/src/syft/VERSION index 384b842bcb3..3a6b1b83053 100644 --- a/packages/syft/src/syft/VERSION +++ b/packages/syft/src/syft/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.5-beta.9" +__version__ = "0.8.5-beta.10" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index b1501b2a5aa..eaa749ec91a 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.5-beta.9" +__version__ = "0.8.5-beta.10" # stdlib from collections.abc import Callable @@ -26,7 +26,7 @@ from .client.user_settings import UserSettings # noqa: F401 from .client.user_settings import settings # noqa: F401 from .custom_worker.config import DockerWorkerConfig # noqa: F401 -from .external import OBLV # noqa: F401 +from .external import OBLV_ENABLED # noqa: F401 from .external import enable_external_lib # noqa: F401 from .node.credentials import SyftSigningKey # noqa: F401 from .node.domain import Domain # noqa: F401 @@ -108,7 +108,7 @@ pass # nosec # For server-side, to enable by environment variable -if OBLV: +if OBLV_ENABLED: enable_external_lib("oblv") diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 2c634158bcd..ff36317238e 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -46,6 +46,7 @@ from ..service.user.user_roles import ServiceRole from ..service.warnings import APIEndpointWarning from ..service.warnings import WarningContext +from ..types.cache_object import CachedSyftObject from ..types.identity import Identity from ..types.syft_object import SYFT_OBJECT_VERSION_2 from ..types.syft_object import SyftBaseObject @@ -55,6 +56,7 @@ from ..types.uid import UID from ..util.autoreload import autoreload_enabled from ..util.telemetry import instrument +from ..util.util import prompt_warning_message from .connection import NodeConnection if TYPE_CHECKING: @@ -753,6 +755,13 @@ def make_call(self, api_call: SyftAPICall) -> Result: result = debox_signed_syftapicall_response(signed_result=signed_result) + if isinstance(result, CachedSyftObject): + if result.error_msg is not None: + prompt_warning_message( + message=f"{result.error_msg}. Loading results from cache." + ) + result = result.result + if isinstance(result, OkErr): if result.is_ok(): result = result.ok() diff --git a/packages/syft/src/syft/external/__init__.py b/packages/syft/src/syft/external/__init__.py index 552a4759d14..b03c6594322 100644 --- a/packages/syft/src/syft/external/__init__.py +++ b/packages/syft/src/syft/external/__init__.py @@ -5,10 +5,12 @@ # stdlib import importlib import os +from typing import Any # relative from ..service.response import SyftError from ..service.response import SyftSuccess +from ..service.service import AbstractService from ..util.util import str_to_bool # Contains all the external libraries that Syft supports. @@ -16,7 +18,7 @@ # if the external library is not installed, we prompt the user # to install it with the pip package name. -OBLV = str_to_bool(os.getenv("OBLV_ENABLED", "false")) +OBLV_ENABLED = str_to_bool(os.getenv("OBLV_ENABLED", "false")) EXTERNAL_LIBS = { "oblv": { @@ -26,6 +28,15 @@ } +def OblvServiceProvider(*args: Any, **kwargs: Any) -> type[AbstractService] | None: + if OBLV_ENABLED: + # relative + from .oblv.oblv_service import OblvService + + return OblvService(*args, **kwargs) + return None + + def package_exists(package_name: str) -> bool: try: importlib.import_module(package_name) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index cd7ff468491..ec1f9b198aa 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -7,7 +7,6 @@ from datetime import datetime from functools import partial import hashlib -from multiprocessing import current_process import os from pathlib import Path import shutil @@ -35,7 +34,7 @@ from ..client.api import debox_signed_syftapicall_response from ..client.client import SyftClient from ..exceptions.exception import PySyftException -from ..external import OBLV +from ..external import OblvServiceProvider from ..protocol.data_protocol import PROTOCOL_TYPE from ..protocol.data_protocol import get_data_protocol from ..service.action.action_object import Action @@ -97,7 +96,7 @@ from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService from ..service.user.user_stash import UserStash -from ..service.veilid import VEILID_ENABLED +from ..service.veilid import VeilidServiceProvider from ..service.worker.image_registry_service import SyftImageRegistryService from ..service.worker.utils import DEFAULT_WORKER_IMAGE_TAG from ..service.worker.utils import DEFAULT_WORKER_POOL_NAME @@ -289,7 +288,6 @@ def __init__( *, # Trasterisk name: str | None = None, id: UID | None = None, - services: list[type[AbstractService]] | None = None, signing_key: SyftSigningKey | SigningKey | None = None, action_store_config: StoreConfig | None = None, document_store_config: StoreConfig | None = None, @@ -321,68 +319,46 @@ def __init__( # 🟡 TODO 22: change our ENV variable format and default init args to make this # less horrible or add some convenience functions self.dev_mode = dev_mode or get_dev_mode() - if node_uid_env is not None: - self.id = UID.from_string(node_uid_env) - else: - if id is None: - id = UID() - self.id = id - + self.id = UID.from_string(node_uid_env) if node_uid_env else (id or UID()) self.packages = "" + self.processes = processes + self.is_subprocess = is_subprocess + self.name = name or random_name() + self.enable_warnings = enable_warnings + self.in_memory_workers = in_memory_workers + self.node_type = NodeType(node_type) + self.node_side_type = NodeSideType(node_side_type) + self.client_cache: dict = {} + self.peer_client_cache: dict = {} - self.signing_key = None - if signing_key_env is not None: - self.signing_key = SyftSigningKey.from_string(signing_key_env) - else: - if isinstance(signing_key, SigningKey): - signing_key = SyftSigningKey(signing_key=signing_key) - self.signing_key = signing_key + if isinstance(node_type, str): + node_type = NodeType(node_type) + self.node_type = node_type - if self.signing_key is None: - self.signing_key = SyftSigningKey.generate() + if isinstance(node_side_type, str): + node_side_type = NodeSideType(node_side_type) + self.node_side_type = node_side_type - if reset: - self.remove_temp_dir() + skey = None + if signing_key_env: + skey = SyftSigningKey.from_string(signing_key_env) + elif isinstance(signing_key, SigningKey): + skey = SyftSigningKey(signing_key=signing_key) + else: + skey = signing_key + self.signing_key = skey or SyftSigningKey.generate() - self.processes = processes - self.is_subprocess = is_subprocess - self.name = random_name() if name is None else name - services = ( - [ - UserService, - WorkerService, - SettingsService, - ActionService, - LogService, - DatasetService, - UserCodeService, - QueueService, - JobService, - RequestService, - DataSubjectService, - NetworkService, - PolicyService, - NotifierService, - NotificationService, - DataSubjectMemberService, - ProjectService, - EnclaveService, - CodeHistoryService, - MetadataService, - BlobStorageService, - MigrateStateService, - SyftWorkerImageService, - SyftWorkerPoolService, - SyftImageRegistryService, - SyncService, - OutputService, - UserCodeStatusService, - ] - if services is None - else services + self.queue_config = self.create_queue_config( + n_consumers=n_consumers, + create_producer=create_producer, + thread_workers=thread_workers, + queue_port=queue_port, + queue_config=queue_config, ) - self.service_config = ServiceConfigRegistry.get_registered_configs() + # must call before initializing stores + if reset: + self.remove_temp_dir() use_sqlite = local_db or (processes > 0 and not is_subprocess) document_store_config = document_store_config or self.get_default_store( @@ -396,23 +372,7 @@ def __init__( document_store_config=document_store_config, ) - if OBLV: - # relative - from ..external.oblv.oblv_service import OblvService - - services += [OblvService] - create_oblv_key_pair(worker=self) - - if VEILID_ENABLED: - # relative - from ..service.veilid.veilid_service import VeilidService - - services += [VeilidService] - - self.enable_warnings = enable_warnings - self.in_memory_workers = in_memory_workers - - self.services = services + # construct services only after init stores self._construct_services() create_admin_new( # nosec B106 @@ -431,26 +391,9 @@ def __init__( smtp_host=smtp_host, ) - self.peer_client_cache: dict = {} - - if isinstance(node_type, str): - node_type = NodeType(node_type) - self.node_type = node_type - - if isinstance(node_side_type, str): - node_side_type = NodeSideType(node_side_type) - self.node_side_type = node_side_type - self.post_init() - self.create_initial_settings(admin_email=root_email) - self.queue_config = self.create_queue_config( - n_consumers=n_consumers, - create_producer=create_producer, - thread_workers=thread_workers, - queue_port=queue_port, - queue_config=queue_config, - ) + self.create_initial_settings(admin_email=root_email) self.init_queue_manager(queue_config=self.queue_config) @@ -830,18 +773,10 @@ def post_init(self) -> None: node_uid=self.id, user_verify_key=self.verify_key, context=context ) - if UserCodeService in self.services: + if "usercodeservice" in self.service_path_map: user_code_service = self.get_service(UserCodeService) user_code_service.load_user_code(context=context) - if self.is_subprocess or current_process().name != "MainProcess": - # print(f"> Starting Subprocess {self}") - pass - else: - pass - # why would we do this? - # print(f"> {self}") - def reload_user_code() -> None: user_code_service.load_user_code(context=context) @@ -904,59 +839,65 @@ def worker_stash(self) -> WorkerStash: return self.get_service("workerservice").stash def _construct_services(self) -> None: - self.service_path_map = {} - - for service_klass in self.services: - kwargs = {} - if service_klass == ActionService: - kwargs["store"] = self.action_store - store_services = [ - UserService, - WorkerService, - SettingsService, - DatasetService, - UserCodeService, - LogService, - RequestService, - QueueService, - JobService, - DataSubjectService, - NetworkService, - PolicyService, - NotifierService, - NotificationService, - DataSubjectMemberService, - ProjectService, - EnclaveService, - CodeHistoryService, - MetadataService, - BlobStorageService, - MigrateStateService, - SyftWorkerImageService, - SyftWorkerPoolService, - SyftImageRegistryService, - SyncService, - OutputService, - UserCodeStatusService, - ] - - if OBLV: - # relative - from ..external.oblv.oblv_service import OblvService + service_path_map: dict[str, AbstractService] = {} + initialized_services: list[AbstractService] = [] + + # A dict of service and init kwargs. + # - "svc" expects a callable (class or function) + # - The callable must return AbstractService or None + # - "store" expects a store type + # - By default all services get the document store + # - Pass a custom "store" to override this + default_services: list[dict] = [ + {"svc": ActionService, "store": self.action_store}, + {"svc": UserService}, + {"svc": WorkerService}, + {"svc": SettingsService}, + {"svc": DatasetService}, + {"svc": UserCodeService}, + {"svc": LogService}, + {"svc": RequestService}, + {"svc": QueueService}, + {"svc": JobService}, + {"svc": DataSubjectService}, + {"svc": NetworkService}, + {"svc": PolicyService}, + {"svc": NotifierService}, + {"svc": NotificationService}, + {"svc": DataSubjectMemberService}, + {"svc": ProjectService}, + {"svc": EnclaveService}, + {"svc": CodeHistoryService}, + {"svc": MetadataService}, + {"svc": BlobStorageService}, + {"svc": MigrateStateService}, + {"svc": SyftWorkerImageService}, + {"svc": SyftWorkerPoolService}, + {"svc": SyftImageRegistryService}, + {"svc": SyncService}, + {"svc": OutputService}, + {"svc": UserCodeStatusService}, + {"svc": VeilidServiceProvider}, # this is lazy + {"svc": OblvServiceProvider}, # this is lazy + ] - store_services += [OblvService] + for svc_kwargs in default_services: + ServiceCls = svc_kwargs.pop("svc") + svc_kwargs.setdefault("store", self.document_store) - if VEILID_ENABLED: - # relative - from ..service.veilid.veilid_service import VeilidService + svc_instance = ServiceCls(**svc_kwargs) + if not svc_instance: + continue + elif not isinstance(svc_instance, AbstractService): + raise ValueError( + f"Service {ServiceCls.__name__} must be an instance of AbstractService" + ) - store_services += [VeilidService] + service_path_map[ServiceCls.__name__.lower()] = svc_instance + initialized_services.append(ServiceCls) - if service_klass in store_services: - kwargs["store"] = self.document_store # type: ignore[assignment] - self.service_path_map[service_klass.__name__.lower()] = service_klass( - **kwargs - ) + self.services = initialized_services + self.service_path_map = service_path_map def get_service_method(self, path_or_func: str | Callable) -> Callable: if callable(path_or_func): @@ -1096,7 +1037,7 @@ def forward_message( self, api_call: SyftAPICall | SignedSyftAPICall ) -> Result[QueueItem | SyftObject, Err]: node_uid = api_call.message.node_uid - if NetworkService not in self.services: + if "networkservice" not in self.service_path_map: return SyftError( message=( "Node has no network service so we can't " @@ -1565,31 +1506,31 @@ def create_admin_new( return None -def create_oblv_key_pair( - worker: Node, -) -> str | None: - try: - # relative - from ..external.oblv.oblv_keys_stash import OblvKeys - from ..external.oblv.oblv_keys_stash import OblvKeysStash - from ..external.oblv.oblv_service import generate_oblv_key - - oblv_keys_stash = OblvKeysStash(store=worker.document_store) - - if not len(oblv_keys_stash) and worker.signing_key: - public_key, private_key = generate_oblv_key(oblv_key_name=worker.name) - oblv_keys = OblvKeys(public_key=public_key, private_key=private_key) - res = oblv_keys_stash.set(worker.signing_key.verify_key, oblv_keys) - if res.is_ok(): - print("Successfully generated Oblv Key pair at startup") - return res.err() - else: - print(f"Using Existing Public/Private Key pair: {len(oblv_keys_stash)}") - except Exception as e: - print("Unable to create Oblv Keys.", e) - return None - - return None +# def create_oblv_key_pair( +# worker: Node, +# ) -> str | None: +# try: +# # relative +# from ..external.oblv.oblv_keys_stash import OblvKeys +# from ..external.oblv.oblv_keys_stash import OblvKeysStash +# from ..external.oblv.oblv_service import generate_oblv_key + +# oblv_keys_stash = OblvKeysStash(store=worker.document_store) + +# if not len(oblv_keys_stash) and worker.signing_key: +# public_key, private_key = generate_oblv_key(oblv_key_name=worker.name) +# oblv_keys = OblvKeys(public_key=public_key, private_key=private_key) +# res = oblv_keys_stash.set(worker.signing_key.verify_key, oblv_keys) +# if res.is_ok(): +# print("Successfully generated Oblv Key pair at startup") +# return res.err() +# else: +# print(f"Using Existing Public/Private Key pair: {len(oblv_keys_stash)}") +# except Exception as e: +# print("Unable to create Oblv Keys.", e) +# return None + +# return None class NodeRegistry: diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index aca46a853dc..7d73092daa4 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -71,7 +71,7 @@ "ExecutionOutput": { "1": { "version": 1, - "hash": "abb4ce9172fbba0ea03fcbea7addb06176c8dba6dbcb7143cde350617528a5b7", + "hash": "201c8abcb6595a64140ad0c3b058557229c7790a25fb55ed229ae0efcb63ad07", "action": "add" } }, diff --git a/packages/syft/src/syft/service/action/action_service.py b/packages/syft/src/syft/service/action/action_service.py index b75dda52bf8..513ca48ff94 100644 --- a/packages/syft/src/syft/service/action/action_service.py +++ b/packages/syft/src/syft/service/action/action_service.py @@ -302,45 +302,34 @@ def _user_code_execute( context.has_execute_permissions or context.role == ServiceRole.ADMIN ) + input_policy = code_item.get_input_policy(context) + if not override_execution_permission: - input_policy = code_item.get_input_policy(context) if input_policy is None: if not code_item.output_policy_approved: return Err("Execution denied: Your code is waiting for approval") - return Err(f"No input poliicy defined for user code: {code_item.id}") + return Err(f"No input policy defined for user code: {code_item.id}") + + # Filter input kwargs based on policy filtered_kwargs = input_policy.filter_kwargs( kwargs=kwargs, context=context, code_item_id=code_item.id ) - if isinstance(filtered_kwargs, SyftError) or filtered_kwargs.is_err(): + if filtered_kwargs.is_err(): return filtered_kwargs filtered_kwargs = filtered_kwargs.ok() + + # validate input policy + is_approved = input_policy._is_valid( + context=context, + usr_input_kwargs=kwargs, + code_item_id=code_item.id, + ) + if is_approved.is_err(): + return is_approved else: filtered_kwargs = retrieve_from_db(code_item.id, kwargs, context).ok() # update input policy to track any input state - if ( - not override_execution_permission - and code_item.get_input_policy(context) is not None - ): - expected_input_kwargs = set() - for _inp_kwarg in code_item.get_input_policy(context).inputs.values(): # type: ignore - keys = _inp_kwarg.keys() - for k in keys: - if k not in kwargs: - return Err( - f"{code_item.service_func_name}() missing required keyword argument: '{k}'" - ) - expected_input_kwargs.update(keys) - - permitted_input_kwargs = list(filtered_kwargs.keys()) - not_approved_kwargs = set(expected_input_kwargs) - set( - permitted_input_kwargs - ) - if len(not_approved_kwargs) > 0: - return Err( - f"Input arguments: {not_approved_kwargs} to the function are not approved yet." - ) - has_twin_inputs = False real_kwargs = {} diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 062dbc2b424..9c680dd288d 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -540,6 +540,7 @@ def apply_output( context: AuthedServiceContext, outputs: Any, job_id: UID | None = None, + input_ids: dict[str, UID] | None = None, ) -> ExecutionOutput | SyftError: output_policy = self.get_output_policy(context) if output_policy is None: @@ -558,6 +559,7 @@ def apply_output( executing_user_verify_key=self.user_verify_key, job_id=job_id, output_policy_id=output_policy.id, + input_ids=input_ids, ) if isinstance(execution_result, SyftError): return execution_result diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 1d3ca8824b7..0b543ba266d 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -15,6 +15,7 @@ from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject +from ...types.cache_object import CachedSyftObject from ...types.twin_object import TwinObject from ...types.uid import UID from ...util.telemetry import instrument @@ -369,7 +370,7 @@ def is_execution_on_owned_args( @service_method(path="code.call", name="call", roles=GUEST_ROLE_LEVEL) def call( self, context: AuthedServiceContext, uid: UID, **kwargs: Any - ) -> SyftSuccess | SyftError: + ) -> CachedSyftObject | ActionObject | SyftSuccess | SyftError: """Call a User Code Function""" kwargs.pop("result_id", None) result = self._call(context, uid, **kwargs) @@ -409,6 +410,11 @@ def _call( # We do not read from output policy cache if there are mock arguments skip_read_cache = len(self.keep_owned_kwargs(kwargs, context)) > 0 + # Extract ids from kwargs + kwarg2id = map_kwargs_to_id(kwargs) + + input_policy = code.get_input_policy(context) + # Check output policy output_policy = code.get_output_policy(context) if not override_execution_permission: @@ -427,17 +433,39 @@ def _call( ) if not (is_valid := output_policy._is_valid(context)): # type: ignore if len(output_history) > 0 and not skip_read_cache: + last_executed_output = output_history[-1] + # Check if the inputs of the last executed output match + # against the current input + if ( + input_policy is not None + and not last_executed_output.check_input_ids( + kwargs=kwarg2id + ) + ): + inp_policy_validation = input_policy._is_valid( + context, + usr_input_kwargs=kwarg2id, + code_item_id=code.id, + ) + if inp_policy_validation.is_err(): + return inp_policy_validation + result: Result[ActionObject, str] = resolve_outputs( context=context, - output_ids=output_history[-1].output_ids, + output_ids=last_executed_output.output_ids, ) if result.is_err(): return result res = delist_if_single(result.ok()) - return Ok(res) + return Ok( + CachedSyftObject( + result=res, + error_msg=is_valid.message, + ) + ) else: - return is_valid.to_result() + return cast(Err, is_valid.to_result()) return can_execute.to_result() # type: ignore # Execute the code item @@ -445,7 +473,6 @@ def _call( action_service = context.node.get_service("actionservice") - kwarg2id = map_kwargs_to_id(kwargs) result_action_object: Result[ActionObject | TwinObject, str] = ( action_service._user_code_execute( context, code, kwarg2id, result_id=result_id @@ -470,7 +497,10 @@ def _call( # and admins executing on high side (TODO, decide if we want to increment counter) if not skip_fill_cache and output_policy is not None: res = code.apply_output( - context=context, outputs=result, job_id=context.job_id + context=context, + outputs=result, + job_id=context.job_id, + input_ids=kwarg2id, ) if isinstance(res, SyftError): return Err(res.message) @@ -518,6 +548,7 @@ def apply_output( context: AuthedServiceContext, user_code_id: UID, outputs: Any, + input_ids: dict[str, UID] | None = None, job_id: UID | None = None, ) -> ExecutionOutput | SyftError: code_result = self.stash.get_by_uid(context.credentials, user_code_id) @@ -528,7 +559,12 @@ def apply_output( if not code.get_status(context).approved: return SyftError(message="Code is not approved") - res = code.apply_output(context=context, outputs=outputs, job_id=job_id) + res = code.apply_output( + context=context, + outputs=outputs, + job_id=job_id, + input_ids=input_ids, + ) return res diff --git a/packages/syft/src/syft/service/enclave/enclave_service.py b/packages/syft/src/syft/service/enclave/enclave_service.py index 73923ad8bd4..052b81efa32 100644 --- a/packages/syft/src/syft/service/enclave/enclave_service.py +++ b/packages/syft/src/syft/service/enclave/enclave_service.py @@ -96,9 +96,9 @@ def send_user_code_inputs_to_enclave( def get_oblv_service() -> type[AbstractService] | SyftError: # relative - from ...external import OBLV + from ...external import OBLV_ENABLED - if OBLV: + if OBLV_ENABLED: # relative from ...external.oblv.oblv_service import OblvService diff --git a/packages/syft/src/syft/service/output/output_service.py b/packages/syft/src/syft/service/output/output_service.py index 6572cafbe7f..3e8ed9e8ffd 100644 --- a/packages/syft/src/syft/service/output/output_service.py +++ b/packages/syft/src/syft/service/output/output_service.py @@ -44,6 +44,7 @@ class ExecutionOutput(SyncableSyftObject): output_ids: list[UID] | dict[str, UID] | None = None job_link: LinkedObject | None = None created_at: DateTime = DateTime.now() + input_ids: dict[str, UID] | None = None # Required for __attr_searchable__, set by model_validator user_code_id: UID @@ -79,6 +80,7 @@ def from_ids( node_uid: UID, job_id: UID | None = None, output_policy_id: UID | None = None, + input_ids: dict[str, UID] | None = None, ) -> "ExecutionOutput": # relative from ..code.user_code_service import UserCode @@ -111,6 +113,7 @@ def from_ids( job_link=job_link, executing_user_verify_key=executing_user_verify_key, output_policy_id=output_policy_id, + input_ids=input_ids, ) @property @@ -142,6 +145,30 @@ def output_id_list(self) -> list[UID]: return ids return [] + @property + def input_id_list(self) -> list[UID]: + ids = self.input_ids + if isinstance(ids, dict): + return list(ids.values()) + return [] + + def check_input_ids(self, kwargs: dict[str, UID]) -> bool: + """ + Checks the input IDs against the stored input IDs. + + Args: + kwargs (dict[str, UID]): A dictionary containing the input IDs to be checked. + + Returns: + bool: True if the input IDs are valid, False otherwise. + """ + if not self.input_ids: + return True + for key, value in kwargs.items(): # Iterate over items of kwargs dictionary + if key not in self.input_ids or self.input_ids[key] != value: + return False + return True + @property def job_id(self) -> UID | None: return self.job_link.object_uid if self.job_link else None @@ -216,6 +243,7 @@ def create( executing_user_verify_key: SyftVerifyKey, job_id: UID | None = None, output_policy_id: UID | None = None, + input_ids: dict[str, UID] | None = None, ) -> ExecutionOutput | SyftError: output = ExecutionOutput.from_ids( output_ids=output_ids, @@ -224,6 +252,7 @@ def create( node_uid=context.node.id, # type: ignore job_id=job_id, output_policy_id=output_policy_id, + input_ids=input_ids, ) res = self.stash.set(context.credentials, output) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index d0f8b2f7ce2..95dc78241eb 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -18,7 +18,9 @@ # third party from RestrictedPython import compile_restricted +from result import Err from result import Ok +from result import Result # relative from ...abstract_node import AbstractNode @@ -177,8 +179,19 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: init_kwargs = partition_by_node(kwargs) super().__init__(*args, init_kwargs=init_kwargs, **kwargs) + def _is_valid( + self, + context: AuthedServiceContext, + usr_input_kwargs: dict, + code_item_id: UID, + ) -> Result[bool, str]: + raise NotImplementedError + def filter_kwargs( - self, kwargs: dict[Any, Any], context: AuthedServiceContext, code_item_id: UID + self, + kwargs: dict[Any, Any], + context: AuthedServiceContext, + code_item_id: UID, ) -> dict[Any, Any]: raise NotImplementedError @@ -213,7 +226,7 @@ def _inputs_for_context(self, context: ChangeContext) -> dict | SyftError: def retrieve_from_db( code_item_id: UID, allowed_inputs: dict[str, UID], context: AuthedServiceContext -) -> dict: +) -> Result[dict[str, Any], str]: # relative from ...service.action.action_object import TwinMode @@ -239,13 +252,13 @@ def retrieve_from_db( has_permission=True, ) if kwarg_value.is_err(): - return SyftError(message=kwarg_value.err()) + return Err(kwarg_value.err()) code_inputs[var_name] = kwarg_value.ok() elif context.node.node_type == NodeType.ENCLAVE: dict_object = action_service.get(context=root_context, uid=code_item_id) if dict_object.is_err(): - return SyftError(message=dict_object.err()) + return Err(dict_object.err()) for value in dict_object.ok().syft_action_data.values(): code_inputs.update(value) @@ -288,7 +301,7 @@ def allowed_ids_only( if uid != allowed_inputs[key]: raise Exception( - f"Input {type(value)} for {key} not in allowed {allowed_inputs}" + f"Input with uid: {uid} for `{key}` not in allowed inputs: {allowed_inputs}" ) filtered_kwargs[key] = value return filtered_kwargs @@ -301,16 +314,57 @@ class ExactMatch(InputPolicy): __version__ = SYFT_OBJECT_VERSION_2 def filter_kwargs( - self, kwargs: dict[Any, Any], context: AuthedServiceContext, code_item_id: UID - ) -> dict[Any, Any]: - allowed_inputs = allowed_ids_only( - allowed_inputs=self.inputs, kwargs=kwargs, context=context - ) - results = retrieve_from_db( - code_item_id=code_item_id, allowed_inputs=allowed_inputs, context=context - ) + self, + kwargs: dict[Any, Any], + context: AuthedServiceContext, + code_item_id: UID, + ) -> Result[dict[Any, Any], str]: + try: + allowed_inputs = allowed_ids_only( + allowed_inputs=self.inputs, kwargs=kwargs, context=context + ) + + results = retrieve_from_db( + code_item_id=code_item_id, + allowed_inputs=allowed_inputs, + context=context, + ) + except Exception as e: + return Err(str(e)) return results + def _is_valid( + self, + context: AuthedServiceContext, + usr_input_kwargs: dict, + code_item_id: UID, + ) -> Result[bool, str]: + filtered_input_kwargs = self.filter_kwargs( + kwargs=usr_input_kwargs, + context=context, + code_item_id=code_item_id, + ) + + if filtered_input_kwargs.is_err(): + return filtered_input_kwargs + + filtered_input_kwargs = filtered_input_kwargs.ok() + + expected_input_kwargs = set() + for _inp_kwargs in self.inputs.values(): + for k in _inp_kwargs.keys(): + if k not in usr_input_kwargs: + return Err(f"Function missing required keyword argument: '{k}'") + expected_input_kwargs.update(_inp_kwargs.keys()) + + permitted_input_kwargs = list(filtered_input_kwargs.keys()) + not_approved_kwargs = set(expected_input_kwargs) - set(permitted_input_kwargs) + if len(not_approved_kwargs) > 0: + return Err( + f"Input arguments: {not_approved_kwargs} to the function are not approved yet." + ) + return Ok(True) + @serializable() class OutputHistory(SyftObject): diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 4180dd1db10..fea214c4904 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -837,8 +837,16 @@ def accept_by_depositing_result( if isinstance(approved, SyftError): return approved + input_ids = {} + if code.input_policy is not None: + for inps in code.input_policy.inputs.values(): + input_ids.update(inps) + res = api.services.code.apply_output( - user_code_id=code.id, outputs=result, job_id=job.id + user_code_id=code.id, + outputs=result, + job_id=job.id, + input_ids=input_ids, ) if isinstance(res, SyftError): return res diff --git a/packages/syft/src/syft/service/veilid/__init__.py b/packages/syft/src/syft/service/veilid/__init__.py index e07b6b857c9..93f60cd6213 100644 --- a/packages/syft/src/syft/service/veilid/__init__.py +++ b/packages/syft/src/syft/service/veilid/__init__.py @@ -1,7 +1,18 @@ # stdlib import os +from typing import Any # relative from ...util.util import str_to_bool VEILID_ENABLED: bool = str_to_bool(os.environ.get("VEILID_ENABLED", "False")) + + +# Any because circular import +def VeilidServiceProvider(*args: Any, **kwargs: Any) -> Any | None: + if VEILID_ENABLED: + # relative + from .veilid_service import VeilidService + + return VeilidService(*args, **kwargs) + return None diff --git a/packages/syft/src/syft/types/cache_object.py b/packages/syft/src/syft/types/cache_object.py new file mode 100644 index 00000000000..ddee2e32a6d --- /dev/null +++ b/packages/syft/src/syft/types/cache_object.py @@ -0,0 +1,14 @@ +# stdlib +from typing import Any + +# relative +from ..serde.serializable import serializable +from .base import SyftBaseModel + + +@serializable() +class CachedSyftObject(SyftBaseModel): + """This class is used to represent the cached result.""" + + result: Any + error_msg: str | None = None diff --git a/packages/syft/tests/syft/users/user_code_test.py b/packages/syft/tests/syft/users/user_code_test.py index 5703703515c..20d7bc50df4 100644 --- a/packages/syft/tests/syft/users/user_code_test.py +++ b/packages/syft/tests/syft/users/user_code_test.py @@ -60,6 +60,12 @@ def test_user_code(worker) -> None: real_result = result.get() assert isinstance(real_result, int) + # Validate that the result is cached + for _ in range(10): + multi_call_res = guest_client.api.services.code.mock_syft_func() + assert isinstance(result, ActionObject) + assert multi_call_res.get() == result.get() + def test_duplicated_user_code(worker, guest_client: User) -> None: # mock_syft_func() diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index 09b26153b75..01eed9ddbca 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -1,11 +1,11 @@ manifestVersion: 1.0 -syftVersion: 0.8.5-beta.9 -dockerTag: 0.8.5-beta.9 +syftVersion: 0.8.5-beta.10 +dockerTag: 0.8.5-beta.10 images: - - docker.io/openmined/grid-frontend:0.8.5-beta.9 - - docker.io/openmined/grid-backend:0.8.5-beta.9 + - docker.io/openmined/grid-frontend:0.8.5-beta.10 + - docker.io/openmined/grid-backend:0.8.5-beta.10 - docker.io/library/mongo:7.0.4 - docker.io/traefik:v2.10 diff --git a/tox.ini b/tox.ini index d84ecc089bd..58ab2016277 100644 --- a/tox.ini +++ b/tox.ini @@ -797,11 +797,9 @@ commands = [testenv:syft.test.helm] description = Test Helm Chart for Kubernetes -changedir = {toxinidir} -passenv=HOME,USER,EXTERNAL_REGISTRY_USERNAME,EXTERNAL_REGISTRY_PASSWORD +changedir = {toxinidir}/packages/grid +passenv=HOME, USER, EXTERNAL_REGISTRY_USERNAME, EXTERNAL_REGISTRY_PASSWORD allowlist_externals = - grep - sleep bash tox setenv = @@ -811,59 +809,38 @@ setenv = EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:not 10-container-images.ipynb} SYFT_VERSION = {env:SYFT_VERSION:local} EXTERNAL_REGISTRY = {env:EXTERNAL_REGISTRY:k3d-registry.localhost:5800} + ; env vars for dev.k8s.start + CLUSTER_NAME = testdomain + CLUSTER_HTTP_PORT = {env:NODE_PORT:8080} commands = - bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE NODE_PORT=$NODE_PORT NODE_URL=$NODE_URL \ - Excluding notebooks: $EXCLUDE_NOTEBOOKS SYFT_VERSION=$SYFT_VERSION \ - EXTERNAL_REGISTRY=$EXTERNAL_REGISTRY; date" + bash -c "env; date; k3d version" + bash -c "k3d cluster delete ${CLUSTER_NAME} || true" - bash -c "k3d version" + tox -e dev.k8s.start - # Remvoing old clusters and volumes and registry - ; bash -c "docker rm $(docker ps -aq) --force || true" - bash -c "k3d cluster delete syft || true" - bash -c "docker volume rm k3d-syft-images --force || true" - bash -c "k3d registry delete k3d-registry.localhost || true" - - # Creating registry - bash -c '\ - export CLUSTER_NAME=syft CLUSTER_HTTP_PORT=${NODE_PORT} && \ - tox -e dev.k8s.start' - - # Creating registry and cluster - bash -c 'NODE_NAME=syft NODE_PORT=${NODE_PORT} && \ - k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \ - k3d cluster start syft' - - sleep 10 - bash -c "kubectl --context k3d-syft create namespace syft || true" - - # if syft version is local, then install local helm charts - # else install the helm charts from the openmined gh-pages branch bash -c 'if [[ $SYFT_VERSION == "local" ]]; then \ echo "Installing local helm charts"; \ - bash -c "cd packages/grid/helm && helm install --kube-context k3d-syft --namespace syft syft ./syft --set global.useDefaultSecrets=true"; \ + helm install ${CLUSTER_NAME} ./helm/syft -f ./helm/values.dev.yaml --kube-context k3d-${CLUSTER_NAME} --namespace syft --create-namespace; \ else \ echo "Installing helm charts from repo for syft version: ${SYFT_VERSION}"; \ - bash -c "helm repo add openmined https://openmined.github.io/PySyft/helm && helm repo update openmined"; \ - bash -c "helm install --kube-context k3d-syft --namespace syft syft openmined/syft --version=${SYFT_VERSION} --set global.useDefaultSecrets=true"; \ + helm repo add openmined https://openmined.github.io/PySyft/helm; \ + helm repo update openmined; \ + helm install ${CLUSTER_NAME} openmined/syft --version=${SYFT_VERSION} -f ./helm/values.dev.yaml --kube-context k3d-${CLUSTER_NAME} --namespace syft --create-namespace; \ fi' ; wait for everything else to be loaded - bash packages/grid/scripts/wait_for.sh service frontend --context k3d-syft --namespace syft - bash -c '(kubectl logs service/frontend --context k3d-syft --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true' - bash packages/grid/scripts/wait_for.sh service mongo --context k3d-syft --namespace syft - bash packages/grid/scripts/wait_for.sh service backend --context k3d-syft --namespace syft - bash packages/grid/scripts/wait_for.sh service proxy --context k3d-syft --namespace syft - bash -c '(kubectl logs service/backend --context k3d-syft --namespace syft -f &) | grep -q "Application startup complete" || true' - + bash -c './scripts/wait_for.sh service frontend --context k3d-$CLUSTER_NAME --namespace syft' + bash -c '(kubectl logs service/frontend --context k3d-$CLUSTER_NAME --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true' + bash -c './scripts/wait_for.sh service mongo --context k3d-$CLUSTER_NAME --namespace syft' + bash -c './scripts/wait_for.sh service backend --context k3d-$CLUSTER_NAME --namespace syft' + bash -c './scripts/wait_for.sh service proxy --context k3d-$CLUSTER_NAME --namespace syft' + bash -c '(kubectl logs service/backend --context k3d-$CLUSTER_NAME --namespace syft -f &) | grep -q "Application startup complete" || true' # Run Notebook tests tox -e e2e.test.notebook - # Cleanup - bash -c "k3d cluster delete syft || true" - bash -c "docker volume rm k3d-syft-images --force || true" + bash -c "k3d cluster delete ${CLUSTER_NAME} || true" [testenv:syft.test.helm.upgrade] description = Test helm upgrade @@ -925,7 +902,7 @@ commands = [testenv:dev.k8s.start] description = Start local Kubernetes registry & cluster with k3d changedir = {toxinidir} -passenv = * +passenv = HOME, USER setenv = CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev} CLUSTER_HTTP_PORT = {env:CLUSTER_HTTP_PORT:8080} @@ -1017,7 +994,7 @@ commands = ; destroy cluster bash -c '\ rm -rf .devspace; echo ""; \ - k3d cluster delete ${CLUSTER_NAME}' + k3d cluster delete ${CLUSTER_NAME};' [testenv:dev.k8s.destroyall] description = Destroy both local Kubernetes cluster and registry @@ -1103,7 +1080,7 @@ commands = # If the syft version is local install the local version # else install the version of syft specified - bash -c " if [[ $SYFT_VERSION == 'local' ]]; then \ + bash -c "if [[ $SYFT_VERSION == 'local' ]]; then \ echo 'Using local syft'; \ else \ echo 'Installing syft version: ${SYFT_VERSION}'; \