diff --git a/docker/grid.Dockerfile b/docker/grid.Dockerfile index df357610e9e..2d7b9ce59ca 100644 --- a/docker/grid.Dockerfile +++ b/docker/grid.Dockerfile @@ -14,7 +14,7 @@ # $ docker run -it -p 5000:5000 openmined/grid-network ARG VERSION=latest -FROM openmined/syft:$VERSION +FROM syft:$VERSION # envs and args ARG APP diff --git a/docker/gridlocaldb.DockerFile b/docker/gridlocaldb.DockerFile new file mode 100644 index 00000000000..df357610e9e --- /dev/null +++ b/docker/gridlocaldb.DockerFile @@ -0,0 +1,36 @@ +#### INSTRUCTIONS +# BUILD the syft base image first + +# for DEV mode, editable source and hot reloading +# $ docker build -f docker/grid.Dockerfile --build-arg APP=domain --build-arg APP_ENV=dev -t openmined/grid-domain-dev:latest -t openmined/grid-domain-dev:`python VERSION` . +# $ docker run -it -v "`pwd`/packages/grid/apps/domain:/app" -v "`pwd`/packages/syft:/syft" -p 5000:5000 openmined/grid-domain-dev + +# for PROD mode, non editable and smaller +# $ docker build -f docker/grid.Dockerfile --build-arg APP=domain --build-arg APP_ENV=production --build-arg VERSION=`python VERSION` -t openmined/grid-domain:latest -t openmined/grid-domain:`python VERSION` . +# $ docker run -it -p 5000:5000 openmined/grid-domain + +# for PROD mode, non editable and smaller +# $ docker build -f docker/grid.Dockerfile --build-arg APP=network --build-arg APP_ENV=production --build-arg VERSION=`python VERSION` -t openmined/grid-network:latest -t openmined/grid-network:`python VERSION` . +# $ docker run -it -p 5000:5000 openmined/grid-network + +ARG VERSION=latest +FROM openmined/syft:$VERSION + +# envs and args +ARG APP +ARG APP_ENV=production +ENV APP_ENV=$APP_ENV +ENV DATABASE_URL=sqlite:///nodedatabase.db +ENV PORT=5000 + +RUN --mount=type=cache,target=/root/.cache python3 -m pip install poetry + +# copy and setup app +RUN mkdir -p /app +COPY ./packages/grid/apps/$APP /app +RUN cd /app && poetry export -f requirements.txt --output requirements.txt --without-hashes +RUN --mount=type=cache,target=/root/.cache pip install -r /app/requirements.txt +RUN pip install psycopg2-binary + +# run the app +CMD ["bash", "-c", "cd /app && ./run.sh"] diff --git a/packages/grid/README.md b/packages/grid/README.md index 630c1397ab2..f97cf5e8e66 100644 --- a/packages/grid/README.md +++ b/packages/grid/README.md @@ -127,6 +127,8 @@ To start the PyGrid Node manually, run: ``` cd apps/node ./run.sh --id bob --port 5000 --start_local_db + +flask run --host=0.0.0.0 --port=5000 --start_local_db ``` You can pass the arguments or use environment variables to set the network configs. diff --git a/packages/grid/apps/domain/pyproject.toml b/packages/grid/apps/domain/pyproject.toml index 40540ffe2e3..0337b9d9ee6 100644 --- a/packages/grid/apps/domain/pyproject.toml +++ b/packages/grid/apps/domain/pyproject.toml @@ -36,7 +36,6 @@ boto3 = "^1.14.51" textwrap3 = "^0.9.2" requests-toolbelt = "0.9.1" scipy = "^1.6.1" -tenseal = "^0.3.2" untokenize = "^0.1.1" [tool.poetry.dev-dependencies] @@ -49,5 +48,5 @@ flake8 = "^3.8.3" flake8-comprehensions = "^3.2.3" [build-system] -requires = ["poetry>=0.12"] +requires = ["setuptools", "poetry>=0.12"] build-backend = "poetry.masonry.api" diff --git a/packages/grid/apps/domain/run.sh b/packages/grid/apps/domain/run.sh index b75c8f3f00f..e4978645698 100755 --- a/packages/grid/apps/domain/run.sh +++ b/packages/grid/apps/domain/run.sh @@ -1,23 +1,23 @@ #!/bin/bash -if [ "$APP_ENV" = "dev" ]; then - # enable hot reloading - export FLASK_ENV=development - - # use this function as the entry point - APP_SRC=$(pwd)/src - export FLASK_APP=${APP_SRC}/app.py:create_app - - # --start_local_db - export LOCAL_DATABASE=$LOCAL_DATABASE - - # allow app imports from the site-packages - export PYTHONPATH="${PYTHONPATH}:${APP_SRC}" - - # run - flask run --host=0.0.0.0 --port=$PORT -fi +# enable hot reloading +export FLASK_ENV=development -if [ "$APP_ENV" = "production" ]; then - exec gunicorn --chdir ./src -k flask_sockets.worker --bind 0.0.0.0:$PORT wsgi:app "$@" -fi +# use this function as the entry point +APP_SRC=$(pwd)/src +export FLASK_APP=${APP_SRC}/app.py:create_app + +# --start_local_db +export LOCAL_DATABASE=true + +# allow app imports from the site-packages +export PYTHONPATH="${PYTHONPATH}:${APP_SRC}" + +# run +flask run --host=0.0.0.0 --port=5000 + +# fi + +# if [ "$APP_ENV" = "production" ]; then +# exec gunicorn --chdir ./src -k flask_sockets.worker --bind 0.0.0.0:$PORT wsgi:app "$@" +# fi diff --git a/packages/grid/apps/domain/src/main/core/node.py b/packages/grid/apps/domain/src/main/core/node.py index 03112c800af..f566168a634 100644 --- a/packages/grid/apps/domain/src/main/core/node.py +++ b/packages/grid/apps/domain/src/main/core/node.py @@ -105,7 +105,8 @@ def create_domain_app(app, args, testing=False): test_config = None if args.start_local_db: - test_config = {"SQLALCHEMY_DATABASE_URI": "sqlite:///nodedatabase.db"} + print("USING IN MEMORY DB") + test_config = {"SQLALCHEMY_DATABASE_URI": "sqlite:///"} # Bind websocket in Flask app instance sockets = Sockets(app) @@ -139,6 +140,7 @@ def create_domain_app(app, args, testing=False): node = GridDomain(name=args.name) # Set SQLAlchemy configs + print(test_config) set_database_config(app, test_config=test_config) app.app_context().push() db.create_all() diff --git a/packages/grid/apps/domain/src/main/core/nodes/domain.py b/packages/grid/apps/domain/src/main/core/nodes/domain.py index fb4d701ebd7..732eeb210fa 100644 --- a/packages/grid/apps/domain/src/main/core/nodes/domain.py +++ b/packages/grid/apps/domain/src/main/core/nodes/domain.py @@ -47,7 +47,7 @@ from syft.core.node.domain.domain import Domain from syft.grid.connections.http_connection import HTTPConnection import sympc -import tenseal as ts +# import tenseal as ts # grid relative from ..database import db diff --git a/packages/grid/apps/domain/src/main/core/nodes/worker.py b/packages/grid/apps/domain/src/main/core/nodes/worker.py index 261539a0743..14cc8ad1d7f 100644 --- a/packages/grid/apps/domain/src/main/core/nodes/worker.py +++ b/packages/grid/apps/domain/src/main/core/nodes/worker.py @@ -24,7 +24,7 @@ from syft.grid.client.client import connect from syft.grid.client.grid_connection import GridHTTPConnection from syft.grid.connections.http_connection import HTTPConnection -import tenseal as ts +# import tenseal as ts # grid relative from ..database import db diff --git a/packages/grid/apps/domain/src/main/core/services/setup_service.py b/packages/grid/apps/domain/src/main/core/services/setup_service.py index c3fa7ca539e..ee2cbba7432 100644 --- a/packages/grid/apps/domain/src/main/core/services/setup_service.py +++ b/packages/grid/apps/domain/src/main/core/services/setup_service.py @@ -31,7 +31,10 @@ def create_initial_setup( msg: CreateInitialSetUpMessage, node: AbstractNode, verify_key: VerifyKey ) -> CreateInitialSetUpResponse: # Should not run if Domain has an owner + print("AAAA") if len(node.users): + print(node.users.first()) + # print(node.users[0]) raise OwnerAlreadyExistsError _email = msg.content.get("email", None) diff --git a/packages/grid/apps/domain/src/main/routes/setup/routes.py b/packages/grid/apps/domain/src/main/routes/setup/routes.py index c58d1ff4cba..cb57ed9723b 100644 --- a/packages/grid/apps/domain/src/main/routes/setup/routes.py +++ b/packages/grid/apps/domain/src/main/routes/setup/routes.py @@ -22,7 +22,7 @@ def initial_setup(current_user): content = request.get_json() if not content: content = {} - + status_code, response_msg = error_handler( route_logic, 200, CreateInitialSetUpMessage, current_user, content ) diff --git a/packages/syft/examples/experimental/koen/Plan Result Permissions.ipynb b/packages/syft/examples/experimental/koen/Plan Result Permissions.ipynb new file mode 100644 index 00000000000..0674646e545 --- /dev/null +++ b/packages/syft/examples/experimental/koen/Plan Result Permissions.ipynb @@ -0,0 +1,319 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "raised-agriculture", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "divided-woman", + "metadata": {}, + "outputs": [], + "source": [ + "import syft as sy\n", + "import numpy as np\n", + "import torch as th\n", + "from syft import VirtualMachine\n", + "from syft.core.plan.plan_builder import PLAN_BUILDER_VM, make_plan, build_plan_inputs, ROOT_CLIENT\n", + "from syft import logger\n", + "from syft import Plan\n", + "from syft.core.node.common.service.set_result_permission_service import SetResultPermissionMessage\n", + "from syft import serialize, deserialize\n", + "import syft\n", + "logger.remove()" + ] + }, + { + "cell_type": "markdown", + "id": "demographic-jason", + "metadata": {}, + "source": [ + "# Result permission example" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "developing-chorus", + "metadata": {}, + "outputs": [], + "source": [ + "node = VirtualMachine(name=\"alice\")" + ] + }, + { + "cell_type": "markdown", + "id": "specialized-theology", + "metadata": {}, + "source": [ + "We define two client, data scientist (ds) and data owner (owner)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "published-strengthening", + "metadata": {}, + "outputs": [], + "source": [ + "owner = node.get_root_client()\n", + "ds = node.get_client()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "serial-spokesman", + "metadata": {}, + "outputs": [], + "source": [ + "@make_plan\n", + "def train(x = th.rand([64*3, 1, 28, 28])):\n", + " return x+1" + ] + }, + { + "cell_type": "markdown", + "id": "opening-groove", + "metadata": {}, + "source": [ + "We create a dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "expressed-float", + "metadata": {}, + "outputs": [], + "source": [ + "x = th.rand([64*3, 1, 28, 28])\n", + "x_ptr = x.send(owner)" + ] + }, + { + "cell_type": "markdown", + "id": "seasonal-overview", + "metadata": {}, + "source": [ + "And let the DS search for a pointer to the data (it will not have read access)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "improving-donna", + "metadata": {}, + "outputs": [], + "source": [ + "x_ds = ds.store[0]" + ] + }, + { + "cell_type": "markdown", + "id": "cleared-capability", + "metadata": {}, + "source": [ + "The ds in turn create a plan, which he wants to apply on the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dominant-siemens", + "metadata": {}, + "outputs": [], + "source": [ + "train_ptr = train.send(ds)" + ] + }, + { + "cell_type": "markdown", + "id": "needed-statement", + "metadata": {}, + "source": [ + "We can execute the plan, but we cannot download the result" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "aerial-castle", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No permissions\n" + ] + } + ], + "source": [ + "try:\n", + " res_ptr = train_ptr(x=x_ptr)\n", + " res_ptr.get()\n", + "except:\n", + " print(\"No permissions\")\n", + "else:\n", + " raise ValueError(\"You should not have permissions to download\")" + ] + }, + { + "cell_type": "markdown", + "id": "curious-glucose", + "metadata": {}, + "source": [ + "Now, the DO finds a pointer to plan, and allows execution on x" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "becoming-baking", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'success'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_ptr_do = [y for y in owner.store if y.__name__ == \"PlanPointer\"][0]\n", + "train_ptr_do.set_result_permissions(\"__call__\", {\"x\": x_ptr}, ds.verify_key)" + ] + }, + { + "cell_type": "markdown", + "id": "turkish-replication", + "metadata": {}, + "source": [ + "And we can verify that the result is the same" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "operational-anaheim", + "metadata": {}, + "outputs": [], + "source": [ + "res_ptr = train_ptr(x=x_ptr)\n", + "result, = res_ptr.get()\n", + "assert th.equal(result, x+1)" + ] + }, + { + "cell_type": "markdown", + "id": "paperback-finger", + "metadata": {}, + "source": [ + "# serialization" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "meaning-nelson", + "metadata": {}, + "outputs": [], + "source": [ + "msg = SetResultPermissionMessage(syft.core.common.UID(), {\"x\": train_ptr.id_at_location}, \"__call__\", ds.verify_key,\n", + " owner.address, owner.address)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "private-providence", + "metadata": {}, + "outputs": [], + "source": [ + "pb = serialize(msg)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "republican-framework", + "metadata": {}, + "outputs": [], + "source": [ + "reconstructed_message = deserialize(pb)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "short-gibson", + "metadata": {}, + "outputs": [], + "source": [ + "r = reconstructed_message\n", + "\n", + "assert r.inputs == msg.inputs\n", + "assert r.id_at_location == msg.id_at_location\n", + "assert r.method_name == msg.method_name\n", + "assert r.verify_key == msg.verify_key\n", + "assert r.address == msg.address\n", + "assert r.reply_to == msg.reply_to" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "northern-slave", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/syft/proto/core/node/common/service/set_result_permission_service.proto b/packages/syft/proto/core/node/common/service/set_result_permission_service.proto new file mode 100644 index 00000000000..eb71559c527 --- /dev/null +++ b/packages/syft/proto/core/node/common/service/set_result_permission_service.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package syft.core.node.common.action; + +import "proto/core/common/common_object.proto"; +import "proto/core/io/address.proto"; + +message SetResultPermissionMessage { + syft.core.common.UID id_at_location = 1; + syft.core.io.Address address = 2; + syft.core.io.Address reply_to = 3; + syft.core.common.UID msg_id = 4; + map inputs = 5; + string method_name = 6; + bytes target_verify_key = 7; +} + +message SetResultPermissionAnswerMessage { + syft.core.io.Address address = 1; + syft.core.common.UID msg_id = 2; + string status_message = 3; +} \ No newline at end of file diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 21f53ceede9..bb19b96de96 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -95,6 +95,7 @@ libs = python-dp statsmodels tenseal + transformers>=4.7.0 xgboost>=1.4 zksk diff --git a/packages/syft/src/syft/core/node/common/action/run_class_method_action.py b/packages/syft/src/syft/core/node/common/action/run_class_method_action.py index 1a89150d881..4352c389331 100644 --- a/packages/syft/src/syft/core/node/common/action/run_class_method_action.py +++ b/packages/syft/src/syft/core/node/common/action/run_class_method_action.py @@ -140,6 +140,12 @@ def execute_action(self, node: AbstractNode, verify_key: VerifyKey) -> None: resolved_kwargs[arg_name] = r_arg.data tag_kwargs[arg_name] = r_arg + method_name = self.path.split(".")[-1] + + if self.has_result_read_permission(resolved_self.result_permissions,method_name, verify_key): + if verify_key not in result_read_permissions: + result_read_permissions[verify_key] = None + ( upcasted_args, upcasted_kwargs, @@ -269,6 +275,12 @@ def _object2proto(self) -> RunClassMethodAction_PB: msg_id=serialize(self.id), ) + def has_result_read_permission(self, permissions, method_name, verify_key): + _args = [a.id_at_location for a in self.args] + _kwargs = {k: v.id_at_location for k, v in self.kwargs.items()} + return any([permission.matches(self._self.id_at_location, verify_key, method_name, _args, _kwargs) for permission in permissions]) + + @staticmethod def _proto2object(proto: RunClassMethodAction_PB) -> "RunClassMethodAction": """Creates a ObjectWithID from a protobuf diff --git a/packages/syft/src/syft/core/node/common/node.py b/packages/syft/src/syft/core/node/common/node.py index a5ceafa2507..3df63766c72 100644 --- a/packages/syft/src/syft/core/node/common/node.py +++ b/packages/syft/src/syft/core/node/common/node.py @@ -66,6 +66,8 @@ from .service.obj_search_service import ImmediateObjectSearchService from .service.repr_service import ReprService from .service.resolve_pointer_type_service import ResolvePointerTypeService +from .service.set_result_permission_service import SetResultPermissionService + # this generic type for Client bound by Client ClientT = TypeVar("ClientT", bound=Client) @@ -208,6 +210,8 @@ def __init__( self.immediate_services_with_reply.append(ImmediateObjectSearchService) self.immediate_services_with_reply.append(GetReprService) self.immediate_services_with_reply.append(ResolvePointerTypeService) + self.immediate_services_with_reply.append(SetResultPermissionService) + # for services which can run at a later time and do not return a reply self.eventual_services_without_reply = list() diff --git a/packages/syft/src/syft/core/node/common/service/set_result_permission_service.py b/packages/syft/src/syft/core/node/common/service/set_result_permission_service.py new file mode 100644 index 00000000000..f2cafa341d4 --- /dev/null +++ b/packages/syft/src/syft/core/node/common/service/set_result_permission_service.py @@ -0,0 +1,136 @@ +# stdlib +from typing import List +from typing import Optional +from typing import Type + +# third party +from google.protobuf.reflection import GeneratedProtocolMessageType +from nacl.signing import VerifyKey + +# syft relative +from ..... import serialize +from .....core.common.serde.serializable import bind_protobuf +from .....proto.core.node.common.service.resolve_pointer_type_service_pb2 import ( + ResolvePointerTypeAnswerMessage as ResolvePointerTypeAnswerMessage_PB, +) +from .....proto.core.node.common.service.set_result_permission_service_pb2 import ( + SetResultPermissionMessage as SetResultPermissionMessage_PB, SetResultPermissionAnswerMessage as SetResultPermissionAnswerMessage_PB +) +from ....common.message import ImmediateSyftMessageWithReply +from ....common.message import ImmediateSyftMessageWithoutReply +from ....common.serde.deserialize import _deserialize +from ....common.uid import UID +from ....io.address import Address +from ...abstract.node import AbstractNode +from .node_service import ImmediateNodeServiceWithReply +from syft.lib.python.dict import Dict +from syft.lib.python.string import String +from syft.core.store.permission.result_permission import ResultPermission + +@bind_protobuf +class SetResultPermissionMessage(ImmediateSyftMessageWithReply): + def __init__( + self, + id_at_location: UID, + inputs, + method_name, + verify_key, + address: Address, + reply_to: Address, + msg_id: Optional[UID] = None, + ): + super().__init__(address=address, msg_id=msg_id, reply_to=reply_to) + self.id_at_location = id_at_location + self.inputs = inputs + self.method_name = method_name + self.verify_key = verify_key + + def _object2proto(self) -> SetResultPermissionMessage_PB: + return SetResultPermissionMessage_PB( + id_at_location=serialize(self.id_at_location), + address=serialize(self.address), + msg_id=serialize(self.id), + reply_to=serialize(self.reply_to), + inputs={k: serialize(v) for k, v in self.inputs.items()}, + method_name=self.method_name, + target_verify_key=bytes(self.verify_key) + ) + + @staticmethod + def _proto2object( + proto: SetResultPermissionMessage_PB, + ) -> "SetResultPermissionMessage": + return SetResultPermissionMessage( + id_at_location=_deserialize(blob=proto.id_at_location), + address=_deserialize(blob=proto.address), + msg_id=_deserialize(blob=proto.msg_id), + reply_to=_deserialize(blob=proto.reply_to), + inputs={k: _deserialize(v) for k,v in proto.inputs.items()}, + method_name=proto.method_name, + verify_key=VerifyKey(proto.target_verify_key) + ) + + @staticmethod + def get_protobuf_schema() -> GeneratedProtocolMessageType: + return SetResultPermissionMessage_PB + + +@bind_protobuf +class SetResultPermissionAnswerMessage(ImmediateSyftMessageWithoutReply): + def __init__( + self, + address: Address, + status_message: str, + msg_id: Optional[UID] = None, + ): + super().__init__(address=address, msg_id=msg_id) + self.status_message=status_message + + def _object2proto(self) -> SetResultPermissionAnswerMessage_PB: + return SetResultPermissionAnswerMessage_PB( + msg_id=serialize(self.id), + address=serialize(self.address), + status_message=self.status_message + ) + + @staticmethod + def _proto2object( + proto: SetResultPermissionAnswerMessage_PB, + ) -> "SetResultPermissionAnswerMessage": + return SetResultPermissionAnswerMessage( + msg_id=_deserialize(blob=proto.msg_id), + address=_deserialize(blob=proto.address), + status_message=proto.status_message + ) + + @staticmethod + def get_protobuf_schema() -> GeneratedProtocolMessageType: + return SetResultPermissionAnswerMessage_PB + + +class SetResultPermissionService(ImmediateNodeServiceWithReply): + @staticmethod + def process( + node: AbstractNode, + msg: SetResultPermissionMessage, + verify_key: Optional[VerifyKey] = None, + ) -> SetResultPermissionAnswerMessage: + if verify_key != node.root_verify_key: + raise ValueError("Root permissions are required in order to set result permissions.") + else: + object = node.store[msg.id_at_location] + result_permission = ResultPermission( + id=msg.id_at_location, + verify_key=msg.verify_key, + method_name=msg.method_name, + kwargs=msg.inputs + ) + if result_permission not in object.result_permissions: + object.result_permissions.append(result_permission) + return SetResultPermissionAnswerMessage( + address=msg.reply_to, msg_id=msg.id, status_message="success" + ) + + @staticmethod + def message_handler_types() -> List[Type[SetResultPermissionMessage]]: + return [SetResultPermissionMessage] \ No newline at end of file diff --git a/packages/syft/src/syft/core/pointer/pointer.py b/packages/syft/src/syft/core/pointer/pointer.py index 830c7262acc..8e2f7513834 100644 --- a/packages/syft/src/syft/core/pointer/pointer.py +++ b/packages/syft/src/syft/core/pointer/pointer.py @@ -585,3 +585,20 @@ def __del__(self) -> None: if self.gc_enabled: self.client.gc.apply(self) + + def set_result_permissions(self, method_name, inputs, verify_key): + from syft.core.node.common.service.set_result_permission_service import SetResultPermissionMessage + + _inputs = {k: v.id_at_location for k, v in inputs.items()} + + obj_msg = SetResultPermissionMessage( + id_at_location=self.id_at_location, + inputs=_inputs, + method_name=method_name, + verify_key=verify_key, + address=self.client.address, + reply_to=self.client.address + ) + + obj = self.client.send_immediate_msg_with_reply(msg=obj_msg).status_message + return obj diff --git a/packages/syft/src/syft/core/store/permission/__init__.py b/packages/syft/src/syft/core/store/permission/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft/src/syft/core/store/permission/result_permission.py b/packages/syft/src/syft/core/store/permission/result_permission.py new file mode 100644 index 00000000000..789996dbd97 --- /dev/null +++ b/packages/syft/src/syft/core/store/permission/result_permission.py @@ -0,0 +1,19 @@ +from nacl.signing import VerifyKey +from typing import List, Dict +from syft.core.common.uid import UID + + +class ResultPermission(): + + def __init__(self, id: UID, verify_key: VerifyKey, method_name: str, args: List[UID]=None, kwargs: Dict[str, UID]=None): + self.id=id + self.verify_key=verify_key + self.method_name=method_name + self.args=args if args is not None else [] + self.kwargs=kwargs if kwargs is not None else dict() + + def __eq__(self, o: 'ResultPermission') -> bool: + return self.id == o.id and self.verify_key == o.verify_key and self.method_name == o.method_name and self.args == o.args and self.kwargs ==o.kwargs + + def matches(self, id, verify_key, method_name, args, kwargs): + return self.id==id and self.verify_key == verify_key and self.method_name==method_name and self.args == args and self.kwargs == kwargs \ No newline at end of file diff --git a/packages/syft/src/syft/core/store/storeable_object.py b/packages/syft/src/syft/core/store/storeable_object.py index 98390b93026..9970f73d469 100644 --- a/packages/syft/src/syft/core/store/storeable_object.py +++ b/packages/syft/src/syft/core/store/storeable_object.py @@ -20,7 +20,8 @@ from ..common.serde.serializable import bind_protobuf from ..common.storeable_object import AbstractStorableObject from ..common.uid import UID - +from .permission.result_permission import ResultPermission +from syft.core.store.permission import result_permission @bind_protobuf class StorableObject(AbstractStorableObject): @@ -59,6 +60,7 @@ def __init__( tags: Optional[List[str]] = None, read_permissions: Optional[dict] = None, search_permissions: Optional[dict] = None, + result_permissions: Optional[List[ResultPermission]]=None ): self.id = id self.data = data @@ -74,6 +76,7 @@ def __init__( # the value is the original request_id to allow lookup later # who are allowed to know that the tensor exists (via search or other means) self.search_permissions: dict = search_permissions if search_permissions else {} + self.result_permissions = result_permission if result_permissions else [] @property def object_type(self) -> str: diff --git a/packages/syft/src/syft/lib/__init__.py b/packages/syft/src/syft/lib/__init__.py index b892cd69ee8..08012764000 100644 --- a/packages/syft/src/syft/lib/__init__.py +++ b/packages/syft/src/syft/lib/__init__.py @@ -27,6 +27,7 @@ from ..lib.remote_dataloader import create_remote_dataloader_ast from ..lib.torch import create_torch_ast from ..lib.torchvision import create_torchvision_ast +from ..lib.transformers import create_ast as create_transformers_ast from ..logger import critical from ..logger import traceback_and_raise from ..logger import warning @@ -229,6 +230,7 @@ def create_lib_ast(client: Optional[Any] = None) -> Globals: # numpy_ast = create_numpy_ast() plan_ast = create_plan_ast(client=client) remote_dataloader_ast = create_remote_dataloader_ast(client=client) + transformers_ast = create_transformers_ast(client=client) lib_ast = Globals(client=client) lib_ast.add_attr(attr_name="syft", attr=python_ast.attrs["syft"]) @@ -238,6 +240,7 @@ def create_lib_ast(client: Optional[Any] = None) -> Globals: lib_ast.syft.core.add_attr( "remote_dataloader", remote_dataloader_ast.syft.core.remote_dataloader ) + lib_ast.add_attr(attr_name="transformers", attr=transformers_ast.attrs["transformers"]) # let the misc creation be always the last, as it needs the full ast solved # to properly generated unions @@ -265,7 +268,6 @@ def create_lib_ast(client: Optional[Any] = None) -> Globals: @wrapt.when_imported("xgboost") @wrapt.when_imported("zksk") @wrapt.when_imported("pytorch_lightning") -@wrapt.when_imported("transformers") def post_import_hook_third_party(module: TypeAny) -> None: """ Note: This needs to be after `lib_ast` because code above uses lib-ast diff --git a/packages/syft/src/syft/proto/core/node/common/service/set_result_permission_service_pb2.py b/packages/syft/src/syft/proto/core/node/common/service/set_result_permission_service_pb2.py new file mode 100644 index 00000000000..ad42ad86287 --- /dev/null +++ b/packages/syft/src/syft/proto/core/node/common/service/set_result_permission_service_pb2.py @@ -0,0 +1,397 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: proto/core/node/common/service/set_result_permission_service.proto +"""Generated protocol buffer code.""" +# third party +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +# syft absolute +from syft.proto.core.common import ( + common_object_pb2 as proto_dot_core_dot_common_dot_common__object__pb2, +) +from syft.proto.core.io import address_pb2 as proto_dot_core_dot_io_dot_address__pb2 + +DESCRIPTOR = _descriptor.FileDescriptor( + name="proto/core/node/common/service/set_result_permission_service.proto", + package="syft.core.node.common.action", + syntax="proto3", + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\nBproto/core/node/common/service/set_result_permission_service.proto\x12\x1csyft.core.node.common.action\x1a%proto/core/common/common_object.proto\x1a\x1bproto/core/io/address.proto"\x8f\x03\n\x1aSetResultPermissionMessage\x12-\n\x0eid_at_location\x18\x01 \x01(\x0b\x32\x15.syft.core.common.UID\x12&\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x15.syft.core.io.Address\x12\'\n\x08reply_to\x18\x03 \x01(\x0b\x32\x15.syft.core.io.Address\x12%\n\x06msg_id\x18\x04 \x01(\x0b\x32\x15.syft.core.common.UID\x12T\n\x06inputs\x18\x05 \x03(\x0b\x32\x44.syft.core.node.common.action.SetResultPermissionMessage.InputsEntry\x12\x13\n\x0bmethod_name\x18\x06 \x01(\t\x12\x19\n\x11target_verify_key\x18\x07 \x01(\x0c\x1a\x44\n\x0bInputsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.syft.core.common.UID:\x02\x38\x01"\x89\x01\n SetResultPermissionAnswerMessage\x12&\n\x07\x61\x64\x64ress\x18\x01 \x01(\x0b\x32\x15.syft.core.io.Address\x12%\n\x06msg_id\x18\x02 \x01(\x0b\x32\x15.syft.core.common.UID\x12\x16\n\x0estatus_message\x18\x03 \x01(\tb\x06proto3', + dependencies=[ + proto_dot_core_dot_common_dot_common__object__pb2.DESCRIPTOR, + proto_dot_core_dot_io_dot_address__pb2.DESCRIPTOR, + ], +) + + +_SETRESULTPERMISSIONMESSAGE_INPUTSENTRY = _descriptor.Descriptor( + name="InputsEntry", + full_name="syft.core.node.common.action.SetResultPermissionMessage.InputsEntry", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="key", + full_name="syft.core.node.common.action.SetResultPermissionMessage.InputsEntry.key", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="value", + full_name="syft.core.node.common.action.SetResultPermissionMessage.InputsEntry.value", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=b"8\001", + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=500, + serialized_end=568, +) + +_SETRESULTPERMISSIONMESSAGE = _descriptor.Descriptor( + name="SetResultPermissionMessage", + full_name="syft.core.node.common.action.SetResultPermissionMessage", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="id_at_location", + full_name="syft.core.node.common.action.SetResultPermissionMessage.id_at_location", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="address", + full_name="syft.core.node.common.action.SetResultPermissionMessage.address", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="reply_to", + full_name="syft.core.node.common.action.SetResultPermissionMessage.reply_to", + index=2, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="msg_id", + full_name="syft.core.node.common.action.SetResultPermissionMessage.msg_id", + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="inputs", + full_name="syft.core.node.common.action.SetResultPermissionMessage.inputs", + index=4, + number=5, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="method_name", + full_name="syft.core.node.common.action.SetResultPermissionMessage.method_name", + index=5, + number=6, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="target_verify_key", + full_name="syft.core.node.common.action.SetResultPermissionMessage.target_verify_key", + index=6, + number=7, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"", + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[ + _SETRESULTPERMISSIONMESSAGE_INPUTSENTRY, + ], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=169, + serialized_end=568, +) + + +_SETRESULTPERMISSIONANSWERMESSAGE = _descriptor.Descriptor( + name="SetResultPermissionAnswerMessage", + full_name="syft.core.node.common.action.SetResultPermissionAnswerMessage", + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name="address", + full_name="syft.core.node.common.action.SetResultPermissionAnswerMessage.address", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="msg_id", + full_name="syft.core.node.common.action.SetResultPermissionAnswerMessage.msg_id", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + _descriptor.FieldDescriptor( + name="status_message", + full_name="syft.core.node.common.action.SetResultPermissionAnswerMessage.status_message", + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=b"".decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=571, + serialized_end=708, +) + +_SETRESULTPERMISSIONMESSAGE_INPUTSENTRY.fields_by_name[ + "value" +].message_type = proto_dot_core_dot_common_dot_common__object__pb2._UID +_SETRESULTPERMISSIONMESSAGE_INPUTSENTRY.containing_type = _SETRESULTPERMISSIONMESSAGE +_SETRESULTPERMISSIONMESSAGE.fields_by_name[ + "id_at_location" +].message_type = proto_dot_core_dot_common_dot_common__object__pb2._UID +_SETRESULTPERMISSIONMESSAGE.fields_by_name[ + "address" +].message_type = proto_dot_core_dot_io_dot_address__pb2._ADDRESS +_SETRESULTPERMISSIONMESSAGE.fields_by_name[ + "reply_to" +].message_type = proto_dot_core_dot_io_dot_address__pb2._ADDRESS +_SETRESULTPERMISSIONMESSAGE.fields_by_name[ + "msg_id" +].message_type = proto_dot_core_dot_common_dot_common__object__pb2._UID +_SETRESULTPERMISSIONMESSAGE.fields_by_name[ + "inputs" +].message_type = _SETRESULTPERMISSIONMESSAGE_INPUTSENTRY +_SETRESULTPERMISSIONANSWERMESSAGE.fields_by_name[ + "address" +].message_type = proto_dot_core_dot_io_dot_address__pb2._ADDRESS +_SETRESULTPERMISSIONANSWERMESSAGE.fields_by_name[ + "msg_id" +].message_type = proto_dot_core_dot_common_dot_common__object__pb2._UID +DESCRIPTOR.message_types_by_name[ + "SetResultPermissionMessage" +] = _SETRESULTPERMISSIONMESSAGE +DESCRIPTOR.message_types_by_name[ + "SetResultPermissionAnswerMessage" +] = _SETRESULTPERMISSIONANSWERMESSAGE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +SetResultPermissionMessage = _reflection.GeneratedProtocolMessageType( + "SetResultPermissionMessage", + (_message.Message,), + { + "InputsEntry": _reflection.GeneratedProtocolMessageType( + "InputsEntry", + (_message.Message,), + { + "DESCRIPTOR": _SETRESULTPERMISSIONMESSAGE_INPUTSENTRY, + "__module__": "proto.core.node.common.service.set_result_permission_service_pb2" + # @@protoc_insertion_point(class_scope:syft.core.node.common.action.SetResultPermissionMessage.InputsEntry) + }, + ), + "DESCRIPTOR": _SETRESULTPERMISSIONMESSAGE, + "__module__": "proto.core.node.common.service.set_result_permission_service_pb2" + # @@protoc_insertion_point(class_scope:syft.core.node.common.action.SetResultPermissionMessage) + }, +) +_sym_db.RegisterMessage(SetResultPermissionMessage) +_sym_db.RegisterMessage(SetResultPermissionMessage.InputsEntry) + +SetResultPermissionAnswerMessage = _reflection.GeneratedProtocolMessageType( + "SetResultPermissionAnswerMessage", + (_message.Message,), + { + "DESCRIPTOR": _SETRESULTPERMISSIONANSWERMESSAGE, + "__module__": "proto.core.node.common.service.set_result_permission_service_pb2" + # @@protoc_insertion_point(class_scope:syft.core.node.common.action.SetResultPermissionAnswerMessage) + }, +) +_sym_db.RegisterMessage(SetResultPermissionAnswerMessage) + + +_SETRESULTPERMISSIONMESSAGE_INPUTSENTRY._options = None +# @@protoc_insertion_point(module_scope) diff --git a/tox.ini b/tox.ini index 1556f66140a..5da94490a4b 100644 --- a/tox.ini +++ b/tox.ini @@ -106,9 +106,10 @@ commands = description = Integration Tests for Syft and Grid basepython = python3.9 deps = - -r{toxinidir}/packages/syft/requirements.unstable.txt changedir = {toxinidir}/packages/syft commands = + pip install -e {toxinidir}/packages/syft[ci-all] + pip install git+https://github.com/OpenMined/SyMPC@16fa17ab61fc37f0aba2f565bda812ae55eb1cdc pip list python ./scripts/mnist.py pytest -m grid -p no:randomly -p no:xdist