diff --git a/.drone.yml b/.drone.yml index 679c23c30..968da46b1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,24 +3,24 @@ name: sgx-debug-ubuntu-1804 steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DTEST_MODE=ON .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: compile - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make VERBOSE=1 -j2 - name: test - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 environment: AS_ALGO: sgx_epid AS_URL: https://api.trustedservices.intel.com:443 @@ -56,26 +56,26 @@ name: sgx-dcap-debug-ubuntu-1804 steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DTEST_MODE=ON -DDCAP=ON .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: compile - image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - sed -i 's/ias_root_ca_cert/dcap_root_ca_cert/' config/build.config.toml - cd build && make VERBOSE=1 -j2 - name: test - image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-dcap-1.6:0.1.1 environment: AS_ALGO: sgx_ecdsa AS_URL: https://localhost:8080 @@ -123,24 +123,24 @@ name: sgx-release-ubuntu-1804 steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DCMAKE_BUILD_TYPE=Release -DTEST_MODE=OFF .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: compile - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make VERBOSE=1 -j2 - name: test - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 privileged: true environment: AS_ALGO: sgx_epid @@ -176,24 +176,24 @@ name: sim-debug-ubuntu-1804 steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DSGX_SIM_MODE=ON -DTEST_MODE=ON .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: compile - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make VERBOSE=1 -j2 - name: test - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 environment: AS_ALGO: sgx_epid AS_URL: https://api.trustedservices.intel.com:443 @@ -215,24 +215,24 @@ name: sim-release-ubuntu-1804 steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DCMAKE_BUILD_TYPE=Release -DSGX_SIM_MODE=ON -DTEST_MODE=OFF .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: compile - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make VERBOSE=1 -j2 - name: test - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make run-examples @@ -413,19 +413,19 @@ name: lint steps: - name: prepare - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - . /opt/sgxsdk/environment - mkdir -p build - cd build && cmake -DRUSTFLAGS="-D warnings" -DTEST_MODE=ON .. - name: check - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make check - name: clippy - image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 + image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 commands: - . /root/.cargo/env - cd build && make CLP=1 @@ -440,24 +440,24 @@ node: # steps: # - name: prepare -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # commands: # - . /root/.cargo/env # - mkdir -p build # - cd build && cmake -DCMAKE_BUILD_TYPE=DEBUG -DCOV=ON -DTEST_MODE=ON .. # - name: check -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # commands: # - . /root/.cargo/env # - cd build && make check # - name: compile -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # commands: # - . /root/.cargo/env # - export RUSTFLAGS="-D warnings" # - cd build && make VERBOSE=1 -j2 # - name: test -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # environment: # AS_ALGO: sgx_epid # AS_URL: https://api.trustedservices.intel.com:443 @@ -476,7 +476,7 @@ node: # - cd build && make run-tests # - name: coverage # failure: ignore -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # commands: # - cd build && make cov # - bash -c "bash <(curl -s https://codecov.io/bash) -f intermediate/cov.info" @@ -508,13 +508,13 @@ node: # steps: # - name: prepare -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # commands: # - . /root/.cargo/env # - mkdir -p build # - cd build && cmake .. # - name: doc -# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.0 +# image: teaclave/teaclave-build-ubuntu-1804-sgx-2.9.1:0.1.1 # failure: ignore # commands: # - . /root/.cargo/env diff --git a/COMMUNITY.md b/COMMUNITY.md index c51d41e9d..8755761d5 100644 --- a/COMMUNITY.md +++ b/COMMUNITY.md @@ -30,7 +30,8 @@ Follow [@ApacheTeaclave](https://twitter.com/ApacheTeaclave). Teaclave is open source in [The Apache Way](https://www.apache.org/theapacheway/), we aim to create a project that is maintained and owned by the community. All -kinds of contributions are welcome. +kinds of contributions are welcome. Read this [document](CONTRIBUTING.md) to +learn more about how to contribute. Huge thanks to our [contributors](CONTRIBUTORS.md). ## Organizations and Projects diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..3a3ea988f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +--- +permalink: /contributing +--- + +# Contributing to Teaclave + +As an open-source community, we welcome all kinds of contributions. You can +contribute to Teaclave in many ways: reporting issues, requesting new features, +proposing better designs, fixing bugs, implementing functions, improving +documents, trying novel research ideas or even by simply using and promoting +this project. + +## Submit Issues + +We prefer to use GitHub issues for almost everything about the project +development such as issues tracking, features, design proposals, announcements, +community communications, etc. Free feel to open an issue if you meet bugs or +want to propose features. + +## Send Pull Requests + +This is a basic instruction to send a pull request to Teaclave. + +1. Fork the repository on GitHub. +2. Create a new branch for the feature or bugfix. +3. Make changes. +4. Test. The `make run-tests` command will run all test case. +5. Make sure to format and lint the code. You can use `make format` to format + code inplace, and `make CLI=1` to lint Rust code with Rust clippy. +6. Commit/push the changes and send a pull request on GitHub. Please kindly + write some background and details for this PR (we also provide a PR template + to guild you with writing a high-quality pull request). diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d52816fdf..98c06aba1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -67,6 +67,7 @@ List of external contributors of Teaclave and Teaclave SGX SDK (in alphabetical - [garbageslam](https://github.com/garbageslam) - [lhf](https://github.com/EighteenZi) - [luoyanhua2011](https://github.com/luoyanhua2011) + - [lyj](https://github.com/lengyijun) - [piotr-roslaniec](https://github.com/piotr-roslaniec) - [volcano](https://github.com/volcano0dr) - [zEqueue](https://github.com/z1queue) diff --git a/README.md b/README.md index 56774d13f..2dbb2f7dd 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,13 @@ platform, making computation on privacy-sensitive data safe and simple. - [Attestation](attestation) - [Built-in Functions](function) +- [Client SDK](sdk) - [Command Line Tool](cli) - [Configurations in Teaclave](config) - [Data Center Attestation Service](dcap) - [Dockerfile and Compose File](docker) - [Examples](examples) +- [Executor Runtime](runtime) - [File Agent](file_agent) - [Function Executors](executor) - [Keys and Certificates](keys) @@ -65,7 +67,8 @@ platform, making computation on privacy-sensitive data safe and simple. Teaclave is open source in [The Apache Way](https://www.apache.org/theapacheway/), we aim to create a project that is maintained and owned by the community. All -kinds of contributions are welcome. Thanks to our [contributors](CONTRIBUTORS.md). +kinds of contributions are welcome. Read this [document](CONTRIBUTING.md) to +learn more about how to contribute. Thanks to our [contributors](CONTRIBUTORS.md). ## Community diff --git a/cmake/UtilTargets.cmake b/cmake/UtilTargets.cmake index 5de4258b2..e5e1ae3fe 100644 --- a/cmake/UtilTargets.cmake +++ b/cmake/UtilTargets.cmake @@ -9,11 +9,19 @@ add_custom_target( format COMMAND rustup component add rustfmt --toolchain ${RUSTUP_TOOLCHAIN} COMMAND - RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} find ${TEACLAVE_PROJECT_ROOT} -path - ${TEACLAVE_PROJECT_ROOT}/third_party -prune -o -path - ${TEACLAVE_PROJECT_ROOT}/.git -prune -o -path ${TEACLAVE_BUILD_ROOT} -prune + RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} find ${TEACLAVE_PROJECT_ROOT} + -path ${TEACLAVE_PROJECT_ROOT}/third_party -prune -o + -path ${TEACLAVE_PROJECT_ROOT}/.git -prune -o + -path ${TEACLAVE_BUILD_ROOT} -prune -o -name "*.rs" -exec rustfmt {} + - COMMENT "Formating every .rs file" + COMMAND + find ${TEACLAVE_PROJECT_ROOT} + -path ${TEACLAVE_PROJECT_ROOT}/third_party -prune -o + -path ${TEACLAVE_PROJECT_ROOT}/.git -prune -o + -path ${TEACLAVE_PROJECT_ROOT}/services/access_control -prune -o + -path ${TEACLAVE_BUILD_ROOT} -prune + -o -name "*.py" -exec yapf -i {} + + COMMENT "Formating every .rs and .py file with rustfmt and yapf" DEPENDS prep) add_custom_target( @@ -24,7 +32,14 @@ add_custom_target( ${TEACLAVE_PROJECT_ROOT}/third_party -prune -o -path ${TEACLAVE_PROJECT_ROOT}/.git -prune -o -path ${TEACLAVE_BUILD_ROOT} -prune -o -name "*.rs" -exec rustfmt --check {} + - COMMENT "Checking the format of every .rs file" + COMMAND + find ${TEACLAVE_PROJECT_ROOT} + -path ${TEACLAVE_PROJECT_ROOT}/third_party -prune -o + -path ${TEACLAVE_PROJECT_ROOT}/.git -prune -o + -path ${TEACLAVE_PROJECT_ROOT}/services/access_control -prune -o + -path ${TEACLAVE_BUILD_ROOT} -prune + -o -name "*.py" -exec yapf -d {} + + COMMENT "Checking the format of every .rs and .py file with rustfmt and yapf" DEPENDS prep) if(TEST_MODE) diff --git a/cmake/scripts/test.sh b/cmake/scripts/test.sh index 64859a030..87e48a83d 100755 --- a/cmake/scripts/test.sh +++ b/cmake/scripts/test.sh @@ -162,6 +162,7 @@ run_examples() { python3 mesapy_echo.py python3 builtin_gbdt_train.py python3 builtin_online_decrypt.py + python3 builtin_private_join_and_compute.py popd # kill all background services diff --git a/docker/build.ubuntu-1804.sgx-2.9.1.Dockerfile b/docker/build.ubuntu-1804.sgx-2.9.1.Dockerfile index 74d5893e7..f9a6266ab 100644 --- a/docker/build.ubuntu-1804.sgx-2.9.1.Dockerfile +++ b/docker/build.ubuntu-1804.sgx-2.9.1.Dockerfile @@ -78,7 +78,7 @@ RUN apt-get update && apt-get install -q -y \ curl \ python3-pip -RUN pip3 install pyopenssl toml cryptography +RUN pip3 install pyopenssl toml cryptography yapf # clean up apt caches diff --git a/docker/build.ubuntu-1804.sgx-dcap-1.6.Dockerfile b/docker/build.ubuntu-1804.sgx-dcap-1.6.Dockerfile index 10eb98231..e7c0c4ed4 100644 --- a/docker/build.ubuntu-1804.sgx-dcap-1.6.Dockerfile +++ b/docker/build.ubuntu-1804.sgx-dcap-1.6.Dockerfile @@ -81,7 +81,7 @@ RUN apt-get update && apt-get install -q -y \ iproute2 \ python3-pip -RUN pip3 install pyopenssl toml cryptography +RUN pip3 install pyopenssl toml cryptography yapf # clean up apt caches diff --git a/docs/how-to-add-your-function.md b/docs/how-to-add-your-function.md new file mode 100644 index 000000000..ee0069e87 --- /dev/null +++ b/docs/how-to-add-your-function.md @@ -0,0 +1,61 @@ +--- +permalink: /docs/how-to-add-your-function +--- + +# Implement your own function + +Example: function/src/private_join_and_compute.rs + +Currently, Teaclave supports two kinds of executors: native functions and Python +functions. In order to support better performance, you can implement you own +functions in Rust. + +## Define function with rust +You need to write down the name of your function and implement the +`run` method. `run` method is the main body of your function. It +takes input from `FunctionArguments` and `FunctionRuntime`. +```rust +#[derive(Default)] +pub struct PrivateJoinAndCompute; +impl PrivateJoinAndCompute { + pub const NAME: &'static str = "builtin-private-join-and-compute"; + pub fn new() -> Self { + Default::default() + } + pub fn run( + &self, + arguments: FunctionArguments, + runtime: FunctionRuntime, + ) -> anyhow::Result { +} +``` + +To implement the `run` method, you may need to read the argument from `FunctionArguments`. +Before that, you need to define the structure of the `FunctionArguments` and convert the +`Json` string into the structure. +```rust +#[derive(serde::Deserialize)] +struct PrivateJoinAndComputeArguments { + num_user: usize, // Number of users in the multiple party computation +} +``` +Here we only pass one parameter into the function, you can read the parameter with the +following code. +```rust +let args = PrivateJoinAndComputeArguments::try_from(arguments)?; +let num_user = args.num_user; +``` +You can also read or write files by `FunctionRuntime`. To open a file, you need to +call the `open_input` method. To write a file, you need to call the `create_output` +method. + +## Rigister your function +After you implement your function, you need to register your function +in [`builtin.rs`](https://github.com/apache/incubator-teaclave/blob/master/executor/src/builtin.rs). +As the source code of the builtin function is conditionally compiled using the attributes cfg. + You also need to update the `Cargo.toml` file. + +## Write the client +Currently, you can implement you client in Rust or Python. Examples can be found under +the `examples/python` and `tests/functional/enclave/src/end_to_end` directories. + diff --git a/examples/python/builtin_echo.py b/examples/python/builtin_echo.py index d9569b2d1..f60fd9a52 100644 --- a/examples/python/builtin_echo.py +++ b/examples/python/builtin_echo.py @@ -15,10 +15,9 @@ def __init__(self, user_id, user_password): self.user_password = user_password def echo(self, message="Hello, Teaclave!"): - channel = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() - client = AuthenticationClient(channel) + client = AuthenticationService( + AUTHENTICATION_SERVICE_ADDRESS, AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() print("[+] registering user") client.user_register(self.user_id, self.user_password) @@ -26,11 +25,11 @@ def echo(self, message="Hello, Teaclave!"): print("[+] login") token = client.user_login(self.user_id, self.user_password) - channel = FrontendService(FRONTEND_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() + client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() metadata = {"id": self.user_id, "token": token} - client = FrontendClient(channel, metadata) + client.metadata = metadata print("[+] registering function") function_id = client.register_function( diff --git a/examples/python/builtin_gbdt_train.py b/examples/python/builtin_gbdt_train.py index 7d727154c..e565ba132 100644 --- a/examples/python/builtin_gbdt_train.py +++ b/examples/python/builtin_gbdt_train.py @@ -3,46 +3,22 @@ import sys from teaclave import (AuthenticationService, FrontendService, - AuthenticationClient, FrontendClient) + AuthenticationClient, FrontendClient, FunctionInput, + FunctionOutput, OwnerList, DataMap) from utils import (AUTHENTICATION_SERVICE_ADDRESS, FRONTEND_SERVICE_ADDRESS, AS_ROOT_CA_CERT_PATH, ENCLAVE_INFO_PATH, USER_ID, USER_PASSWORD) -class FunctionInput: - def __init__(self, name, description): - self.name = name - self.description = description - - -class FunctionOutput: - def __init__(self, name, description): - self.name = name - self.description = description - - -class OwnerList: - def __init__(self, data_name, uids): - self.data_name = data_name - self.uids = uids - - -class DataList: - def __init__(self, data_name, data_id): - self.data_name = data_name - self.data_id = data_id - - class BuiltinGbdtExample: def __init__(self, user_id, user_password): self.user_id = user_id self.user_password = user_password def gbdt(self): - channel = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() - client = AuthenticationClient(channel) + client = AuthenticationService( + AUTHENTICATION_SERVICE_ADDRESS, AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() print("[+] registering user") client.user_register(self.user_id, self.user_password) @@ -50,11 +26,11 @@ def gbdt(self): print("[+] login") token = client.user_login(self.user_id, self.user_password) - channel = FrontendService(FRONTEND_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() + client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() metadata = {"id": self.user_id, "token": token} - client = FrontendClient(channel, metadata) + client.metadata = metadata print("[+] registering function") function_id = client.register_function( @@ -110,8 +86,8 @@ def gbdt(self): print("[+] assigning data to task") client.assign_data_to_task( - task_id, [DataList("training_data", training_data_id)], - [DataList("trained_model", output_model_id)]) + task_id, [DataMap("training_data", training_data_id)], + [DataMap("trained_model", output_model_id)]) print("[+] approving task") client.approve_task(task_id) diff --git a/examples/python/builtin_online_decrypt.py b/examples/python/builtin_online_decrypt.py index d14cf7a94..e660e57fb 100644 --- a/examples/python/builtin_online_decrypt.py +++ b/examples/python/builtin_online_decrypt.py @@ -16,10 +16,9 @@ def __init__(self, user_id, user_password): self.user_password = user_password def decrypt(self, key, nonce, encrypted_data, algorithm): - channel = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() - client = AuthenticationClient(channel) + client = AuthenticationService( + AUTHENTICATION_SERVICE_ADDRESS, AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() print("[+] registering user") client.user_register(self.user_id, self.user_password) @@ -27,11 +26,11 @@ def decrypt(self, key, nonce, encrypted_data, algorithm): print("[+] login") token = client.user_login(self.user_id, self.user_password) - channel = FrontendService(FRONTEND_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() + client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() metadata = {"id": self.user_id, "token": token} - client = FrontendClient(channel, metadata) + client.metadata = metadata print("[+] registering function") function_id = client.register_function( diff --git a/examples/python/builtin_private_join_and_compute.py b/examples/python/builtin_private_join_and_compute.py new file mode 100644 index 000000000..a142bd6e3 --- /dev/null +++ b/examples/python/builtin_private_join_and_compute.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 + +import sys + +from teaclave import (AuthenticationService, FrontendService, + AuthenticationClient, FrontendClient) +from utils import (AUTHENTICATION_SERVICE_ADDRESS, FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, ENCLAVE_INFO_PATH, USER_ID, + USER_PASSWORD) + +# In the below example, user 3 creates the task and user 0, 1, 2 upload their private data. +# Then user 3 invokes the task and user 0, 1, 2 get the result. + +USER3_ID = "user3" +USER3_PASSWORD = "password" + +USER0_ID = "user0" +USER0_PASSWORD = "password" +USER0_input_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_data/bank_a.enc" +USER0_output_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_results/user0_output.enc" +USER0_input_cmac = "7884a62894e7be50b9795ba22ce5ee7f" +USER0_key = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + +USER1_ID = "user1" +USER1_PASSWORD = "password" +USER1_input_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_data/bank_b.enc" +USER1_output_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_results/user1_output.enc" +USER1_input_cmac = "75b8e931887bd57564d93df31c282bb9" +USER1_key = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + +USER2_ID = "user2" +USER2_PASSWORD = "password" +USER2_input_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_data/bank_c.enc" +USER2_output_url = "http://localhost:6789/fixtures/functions/private_join_and_compute/three_party_results/user2_output.enc" +USER2_input_cmac = "35acf29139485067d1ae6212c0577b43" +USER2_key = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + +class FunctionInput: + def __init__(self, name, description): + self.name = name + self.description = description + + +class FunctionOutput: + def __init__(self, name, description): + self.name = name + self.description = description + + +class OwnerList: + def __init__(self, data_name, uids): + self.data_name = data_name + self.uids = uids + + +class DataList: + def __init__(self, data_name, data_id): + self.data_name = data_name + self.data_id = data_id + + +class ConfigClient: + def __init__(self, user_id, user_password): + self.user_id = user_id + self.user_password = user_password + self.client = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() + print("[+] registering user") + self.client.user_register(self.user_id, self.user_password) + print("[+] login") + token = self.client.user_login(self.user_id, self.user_password) + self.client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() + metadata = {"id": self.user_id, "token": token} + self.client.metadata = metadata + + def set_task(self): + client = self.client + + print("[+] registering function") + + function_id = client.register_function( + name="builtin-private-join-and-compute", + description="Native Private Join And Compute", + executor_type="builtin", + arguments=["num_user"], + inputs=[ + FunctionInput("input_data0", "Bank A data file."), + FunctionInput("input_data1", "Bank B data file."), + FunctionInput("input_data2", "Bank C data file.") + ], + outputs=[ + FunctionOutput("output_data0", "Output data."), + FunctionOutput("output_data1", "Output data."), + FunctionOutput("output_data2", "Output date.") + ]) + + print("[+] creating task") + task_id = client.create_task( + function_id=function_id, + function_arguments=({ + "num_user": 3, + }), + executor="builtin", + inputs_ownership=[ + OwnerList("input_data0", [USER0_ID]), + OwnerList("input_data1", [USER1_ID]), + OwnerList("input_data2", [USER2_ID]) + ], + outputs_ownership=[ + OwnerList("output_data0", [USER0_ID]), + OwnerList("output_data1", [USER1_ID]), + OwnerList("output_data2", [USER2_ID]) + ]) + + return task_id + + def run_task(self, task_id): + client = self.client + + client.approve_task(task_id) + print("[+] invoking task") + client.invoke_task(task_id) + + print("[+] getting result") + result = client.get_task_result(task_id) + print("[+] done") + + return bytes(result) + + +class DataClient: + def __init__(self, user_id, user_password): + self.user_id = user_id + self.user_password = user_password + self.client = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() + print("[+] registering user") + self.client.user_register(self.user_id, self.user_password) + print("[+] login") + token = self.client.user_login(self.user_id, self.user_password) + self.client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() + metadata = {"id": self.user_id, "token": token} + self.client.metadata = metadata + + def register_data(self, task_id, input_url, input_cmac, output_url, + file_key, input_label, output_label): + client = self.client + + print("[+] registering input file") + url = input_url + cmac = input_cmac + schema = "teaclave-file-128" + key = file_key + iv = [] + training_data_id = client.register_input_file(url, schema, key, iv, + cmac) + print("[+] registering output file") + url = output_url + schema = "teaclave-file-128" + key = file_key + iv = [] + output_model_id = client.register_output_file(url, schema, key, iv) + + print("[+] assigning data to task") + client.assign_data_to_task(task_id, + [DataList(input_label, training_data_id)], + [DataList(output_label, output_model_id)]) + + print("[+] data client approving task") + + def approve_task(self, task_id): + client = self.client + client.approve_task(task_id) + + +def main(): + + config_client = ConfigClient(USER3_ID, USER3_PASSWORD) + task_id = config_client.set_task() + + user0 = DataClient(USER0_ID, USER0_PASSWORD) + user0.register_data(task_id, USER0_input_url, USER0_input_cmac, + USER0_output_url, USER0_key, "input_data0", + "output_data0") + + user1 = DataClient(USER1_ID, USER1_PASSWORD) + user1.register_data(task_id, USER1_input_url, USER1_input_cmac, + USER1_output_url, USER1_key, "input_data1", + "output_data1") + + user2 = DataClient(USER2_ID, USER2_PASSWORD) + user2.register_data(task_id, USER2_input_url, USER2_input_cmac, + USER2_output_url, USER2_key, "input_data2", + "output_data2") + + user0.approve_task(task_id) + user1.approve_task(task_id) + user2.approve_task(task_id) + + rt = config_client.run_task(task_id) + print("[+] function return: ", rt) + + +if __name__ == '__main__': + main() diff --git a/examples/python/mesapy_echo.py b/examples/python/mesapy_echo.py index 570adc74f..3f1251234 100644 --- a/examples/python/mesapy_echo.py +++ b/examples/python/mesapy_echo.py @@ -17,10 +17,9 @@ def __init__(self, user_id, user_password): def echo(self, payload_file="mesapy_echo_payload.py", message="Hello, Teaclave!"): - channel = AuthenticationService(AUTHENTICATION_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() - client = AuthenticationClient(channel) + client = AuthenticationService( + AUTHENTICATION_SERVICE_ADDRESS, AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() print("[+] registering user") client.user_register(self.user_id, self.user_password) @@ -28,11 +27,11 @@ def echo(self, print("[+] login") token = client.user_login(self.user_id, self.user_password) - channel = FrontendService(FRONTEND_SERVICE_ADDRESS, - AS_ROOT_CA_CERT_PATH, - ENCLAVE_INFO_PATH).connect() + client = FrontendService(FRONTEND_SERVICE_ADDRESS, + AS_ROOT_CA_CERT_PATH, + ENCLAVE_INFO_PATH).connect().get_client() metadata = {"id": self.user_id, "token": token} - client = FrontendClient(channel, metadata) + client.metadata = metadata print("[+] registering function") diff --git a/executor/Cargo.toml b/executor/Cargo.toml index 19c0e7e24..d1505972b 100644 --- a/executor/Cargo.toml +++ b/executor/Cargo.toml @@ -32,6 +32,7 @@ full_builtin_function = [ "builtin_logistic_regression_predict", "builtin_logistic_regression_train", "builtin_online_decrypt", + "builtin_private_join_and_compute", ] builtin_echo = [] @@ -40,6 +41,7 @@ builtin_gbdt_train = [] builtin_logistic_regression_predict = [] builtin_logistic_regression_train = [] builtin_online_decrypt = [] +builtin_private_join_and_compute = [] [dependencies] log = { version = "0.4.6" } diff --git a/executor/src/builtin.rs b/executor/src/builtin.rs index 990d0147e..ef4c4137e 100644 --- a/executor/src/builtin.rs +++ b/executor/src/builtin.rs @@ -19,7 +19,8 @@ use std::prelude::v1::*; use teaclave_function::{ - Echo, GbdtPredict, GbdtTrain, LogisticRegressionPredict, LogisticRegressionTrain, OnlineDecrypt, + Echo, GbdtPredict, GbdtTrain, LogisticRegressionPredict, LogisticRegressionTrain, + OnlineDecrypt, PrivateJoinAndCompute, }; use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveExecutor}; @@ -51,6 +52,8 @@ impl TeaclaveExecutor for BuiltinFunctionExecutor { } #[cfg(feature = "builtin_online_decrypt")] OnlineDecrypt::NAME => OnlineDecrypt::new().run(arguments, runtime), + #[cfg(feature = "builtin_private_join_and_compute")] + PrivateJoinAndCompute::NAME => PrivateJoinAndCompute::new().run(arguments, runtime), _ => bail!("Function not found."), } } diff --git a/function/src/lib.rs b/function/src/lib.rs index 43189b699..0e8388541 100644 --- a/function/src/lib.rs +++ b/function/src/lib.rs @@ -28,6 +28,7 @@ mod gbdt_train; mod logistic_regression_predict; mod logistic_regression_train; mod online_decrypt; +mod private_join_and_compute; pub use echo::Echo; pub use gbdt_predict::GbdtPredict; @@ -35,6 +36,7 @@ pub use gbdt_train::GbdtTrain; pub use logistic_regression_predict::LogisticRegressionPredict; pub use logistic_regression_train::LogisticRegressionTrain; pub use online_decrypt::OnlineDecrypt; +pub use private_join_and_compute::PrivateJoinAndCompute; #[cfg(feature = "enclave_unit_test")] pub mod tests { @@ -49,6 +51,7 @@ pub mod tests { logistic_regression_train::tests::run_tests(), logistic_regression_predict::tests::run_tests(), online_decrypt::tests::run_tests(), + private_join_and_compute::tests::run_tests(), ) } } diff --git a/function/src/private_join_and_compute.rs b/function/src/private_join_and_compute.rs new file mode 100644 index 000000000..6aa82e62e --- /dev/null +++ b/function/src/private_join_and_compute.rs @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::{bail, Result}; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::fmt; +use std::format; +use std::io::Write; +#[cfg(feature = "mesalock_sgx")] +use std::prelude::v1::*; +use teaclave_types::{FunctionArguments, FunctionRuntime}; + +const IN_DATA: &str = "input_data"; +const OUT_RESULT: &str = "output_data"; + +#[derive(Default)] +pub struct PrivateJoinAndCompute; + +#[derive(serde::Deserialize)] +struct PrivateJoinAndComputeArguments { + num_user: usize, // Number of users in the mutiple party computation +} + +impl TryFrom for PrivateJoinAndComputeArguments { + type Error = anyhow::Error; + + fn try_from(arguments: FunctionArguments) -> Result { + use anyhow::Context; + serde_json::from_str(&arguments.into_string()).context("Cannot deserialize arguments") + } +} + +impl PrivateJoinAndCompute { + pub const NAME: &'static str = "builtin-private-join-and-compute"; + + pub fn new() -> Self { + Default::default() + } + + pub fn run( + &self, + arguments: FunctionArguments, + runtime: FunctionRuntime, + ) -> anyhow::Result { + let args = PrivateJoinAndComputeArguments::try_from(arguments)?; + let num_user = args.num_user; + if num_user < 2 { + bail!("The demo requires at least two parties!"); + } + + let mut output = String::new(); + let data_0 = get_data(0, &runtime)?; + let input_map_0 = parse_input(data_0)?; + let mut res_map: HashMap = input_map_0; + + for i in 1..num_user { + let data = get_data(i, &runtime)?; + let input_map = parse_input(data)?; + res_map = get_intersection_sum(&input_map, &res_map); + } + + for (identity, amount) in res_map { + fmt::write(&mut output, format_args!("{}, {}\n", identity, amount))?; + } + + let output_bytes = output.as_bytes(); + + for i in 0..num_user { + let output_file_name = format!("{}{}", OUT_RESULT, i); + let mut output = runtime.create_output(&output_file_name)?; + output.write_all(&output_bytes)?; + } + + let summary = format!("{} users join the task in total.", num_user); + Ok(summary) + } +} + +fn get_data(user_id: usize, runtime: &FunctionRuntime) -> anyhow::Result> { + let mut data: Vec = Vec::new(); + let input_file_name = format!("{}{}", IN_DATA, user_id); + let mut input_io = runtime.open_input(&input_file_name)?; + input_io.read_to_end(&mut data)?; + Ok(data) +} + +fn get_intersection_sum( + map1: &HashMap, + map2: &HashMap, +) -> HashMap { + let mut res_map: HashMap = HashMap::new(); + for (identity, amount) in map1 { + if map2.contains_key(identity) { + let total = amount + map2[identity]; + res_map.insert(identity.to_owned(), total); + } + } + res_map +} + +fn parse_input(data: Vec) -> anyhow::Result> { + let data_list = String::from_utf8(data)?; + let mut ret: HashMap = HashMap::new(); + for data_item in data_list.split('\n') { + let pair = data_item.trim(); + if pair.len() < 3 { + continue; + } + let kv_pair: Vec<&str> = pair.split(':').collect(); + if kv_pair.len() != 2 { + continue; + } + let identity = kv_pair[0].trim().to_string(); + let amount = match kv_pair[1].trim().parse::() { + Ok(amount) => amount, + Err(_) => continue, + }; + ret.insert(identity, amount); + } + Ok(ret) +} + +#[cfg(feature = "enclave_unit_test")] +pub mod tests { + use super::*; + use serde_json::json; + use std::untrusted::fs; + use teaclave_crypto::*; + use teaclave_runtime::*; + use teaclave_test_utils::*; + use teaclave_types::*; + + pub fn run_tests() -> bool { + run_tests!(test_private_join_and_compute) + } + + fn test_private_join_and_compute() { + let arguments = FunctionArguments::from_json(json!({ + "num_user": 3 + })) + .unwrap(); + + let user0_input = "fixtures/functions/private_join_and_compute/three_party_data/bank_a.txt"; + let user0_output = + "fixtures/functions/private_join_and_compute/three_party_results/user0_output.txt"; + + let user1_input = "fixtures/functions/private_join_and_compute/three_party_data/bank_b.txt"; + let user1_output = + "fixtures/functions/private_join_and_compute/three_party_results/user1_output.txt"; + + let user2_input = "fixtures/functions/private_join_and_compute/three_party_data/bank_c.txt"; + let user2_output = + "fixtures/functions/private_join_and_compute/three_party_results/user2_output.txt"; + + let input_files = StagedFiles::new(hashmap!( + "input_data0" => + StagedFileInfo::new(user0_input, TeaclaveFile128Key::random(), FileAuthTag::mock()), + "input_data1" => + StagedFileInfo::new(user1_input, TeaclaveFile128Key::random(), FileAuthTag::mock()), + "input_data2" => + StagedFileInfo::new(user2_input, TeaclaveFile128Key::random(), FileAuthTag::mock()) + )); + + let output_files = StagedFiles::new(hashmap!( + "output_data0" => + StagedFileInfo::new(user0_output, TeaclaveFile128Key::random(), FileAuthTag::mock()), + "output_data1" => + StagedFileInfo::new(user1_output, TeaclaveFile128Key::random(), FileAuthTag::mock()), + "output_data2" => + StagedFileInfo::new(user2_output, TeaclaveFile128Key::random(), FileAuthTag::mock()) + )); + + let runtime = Box::new(RawIoRuntime::new(input_files, output_files)); + + let summary = PrivateJoinAndCompute::new() + .run(arguments, runtime) + .unwrap(); + + let user0 = fs::read_to_string(&user0_output).unwrap(); + let user1 = fs::read_to_string(&user1_output).unwrap(); + let user2 = fs::read_to_string(&user2_output).unwrap(); + assert_eq!(&user0[..], summary); + assert_eq!(&user1[..], summary); + assert_eq!(&user2[..], summary); + } +} diff --git a/runtime/README.md b/runtime/README.md new file mode 100644 index 000000000..bad651acc --- /dev/null +++ b/runtime/README.md @@ -0,0 +1,14 @@ +--- +permalink: /runtime +--- + +# Executor Runtime + +This directory contains implementations of executor's runtime. The executor +runtime provides interfaces (I/O) between executors (in trusted execution +environment) and external components (in untrusted world like file system). The +interfaces are defined in the `TeaclaveRuntime` traits. Currently, we have two +runtime implementations: `DefaultRuntime` and `RawIoRuntime`. By default, +Teaclave provides a runtime called `DefaultRuntime`, which bridges interfaces to +our secure file system implementation (i.e., *protected file*). While +`RawIoRuntime` is only for debugging, which does not encrypt any I/O. diff --git a/sdk/README.md b/sdk/README.md index 5a4353618..5284ec364 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -1 +1,10 @@ -sdk +--- +permalink: /sdk +--- + +# Client SDK + +This directory provides Teaclave client SDK in different languages. Developers +can uses the SDK to establish trusted channel with Teaclave services, send +requests via RPC, etc. Please refer to the +[document for examples](../examples/README.md) to learn more about the usages. diff --git a/sdk/python/teaclave.py b/sdk/python/teaclave.py index 31d08587f..1da3179b5 100644 --- a/sdk/python/teaclave.py +++ b/sdk/python/teaclave.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +""" +Python package `teaclave` is the client SDK for Python developers, providing +some essential data structures, service, and client classes to establish +trusted TLS channel and communicate with Teaclave services (e.g., the +authentication service and frontend service) through RPC protocols. +""" import struct import json @@ -9,6 +15,8 @@ import ssl import socket +from typing import Tuple, Dict, List, Any + from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -16,83 +24,226 @@ from OpenSSL.crypto import X509Store, X509StoreContext from OpenSSL import crypto +__all__ = [ + 'FrontendClient', 'FrontendService', 'AuthenticationClient', + 'AuthenticationService', 'FunctionInput', 'FunctionOutput', 'OwnerList', + 'DataMap' +] + +Metadata = Dict[str, str] + + +class FunctionInput: + """Function input for registering. + + Args: + name: Name of input data. + description: Description of the input data. + """ + def __init__(self, name: str, description: str): + self.name = name + self.description = description + + +class FunctionOutput: + """Function output for registering. + + Args: + name: Name of output data. + description: Description of the output data. + """ + def __init__(self, name: str, description: str): + self.name = name + self.description = description + + +class OwnerList: + """Defines data ownership. + + Args: + data_name: Name of output data. + uids: A list of user id which own this data. + """ + def __init__(self, data_name: str, uids: List[str]): + self.data_name = data_name + self.uids = uids + -class RequestEncoder(json.JSONEncoder): - def default(self, o): - return o.__dict__ +class DataMap: + """Assign data id to input or output data. + + Args: + data_name: Name of output data. + data_id: Id for the data name. + """ + def __init__(self, data_name, data_id): + self.data_name = data_name + self.data_id = data_id + + +class CryptoInfo: + """Cryptographic information for the input/output data. + + Args: + schema: Encryption algorithms for the input/output data. + key: Key for encryption and decryption, bytes in list. + iv: IV, bytes in list. + """ + def __init__(self, schema: str, key: List[int], iv: List[int]): + self.schema = schema + self.key = key + self.iv = iv class UserRegisterReqeust: - def __init__(self, user_id, user_password): + def __init__(self, user_id: str, user_password: str): self.request = "user_register" self.id = user_id self.password = user_password class UserLoginRequest: - def __init__(self, user_id, user_password): + def __init__(self, user_id: str, user_password: str): self.request = "user_login" self.id = user_id self.password = user_password -class AuthenticationClient: - def __init__(self, channel): - self.channel = channel - - def user_register(self, user_id, user_password): - request = UserRegisterReqeust(user_id, user_password) - write_message(self.channel, request) - return read_message(self.channel) - - def user_login(self, user_id, user_password): - request = UserLoginRequest(user_id, user_password) - write_message(self.channel, request) - response = read_message(self.channel) - return response["content"]["token"] - - class AuthenticationService: - context = ssl._create_unverified_context() - - def __init__(self, address, as_root_ca_cert_path, enclave_info_path): + """ + Establish trusted channel with the authentication service and provide + clients to send request through RPC. + + Args: + address: The address of the remote services in tuple. + as_root_ca_cert_path: Root CA certification of the attestation services + to verify the attestation report. + enclave_info_path: Path of enclave info to verify the remote service in + the attestation report. + """ + _context = ssl._create_unverified_context() + _channel = None + + def __init__(self, address: Tuple[str, int], as_root_ca_cert_path: str, + enclave_info_path: str): self.address = address self.as_root_ca_cert_path = as_root_ca_cert_path self.enclave_info_path = enclave_info_path def connect(self): + """Establish trusted connection and verify remote attestation report. + + Returns: + AuthenticationService: The original object which can be chained + with other methods. + """ sock = socket.create_connection(self.address) - channel = self.context.wrap_socket(sock, - server_hostname=self.address[0]) + channel = self._context.wrap_socket(sock, + server_hostname=self.address[0]) cert = channel.getpeercert(binary_form=True) - verify_report(self.as_root_ca_cert_path, self.enclave_info_path, cert, - "authentication") + _verify_report(self.as_root_ca_cert_path, self.enclave_info_path, cert, + "authentication") - return channel + self._channel = channel + return self -class FrontendService: - context = ssl._create_unverified_context() + def get_client(self): + """Get a client of authentication service to send RPC requests. + + Returns: + AuthenticationClient: Used for send/receive RPC requests. + """ + return AuthenticationClient(self._channel) + + +class AuthenticationClient: + """Client to communicate with the authentication service. - def __init__(self, address, as_root_ca_cert_path, enclave_info_path): + Args: + channel: Trusted TLS socket (verified with remote attestation). + """ + def __init__(self, channel: ssl.SSLSocket): + self.channel = channel + + def user_register(self, user_id: str, user_password: str): + """Register a new user. + + Args: + user_id: User ID. + user_password: Password. + """ + request = UserRegisterReqeust(user_id, user_password) + _write_message(self.channel, request) + _ = _read_message(self.channel) + + def user_login(self, user_id: str, user_password: str) -> str: + """Login and get a session token. + + Args: + user_id: User ID. + user_password: Password. + + Returns: + str: User login token. + """ + request = UserLoginRequest(user_id, user_password) + _write_message(self.channel, request) + response = _read_message(self.channel) + return response["content"]["token"] + + +class FrontendService: + """Establish trusted channel with the frontend service and provide + clients to send request through RPC. + + Args: + address: The address of the remote services in tuple. + as_root_ca_cert_path: Root CA certification of the attestation services + to verify the attestation report. + enclave_info_path: Path of enclave info to verify the remote service in + the attestation report. + """ + _context = ssl._create_unverified_context() + _channel = None + + def __init__(self, address: Tuple[str, int], as_root_ca_cert_path: str, + enclave_info_path: str): self.address = address self.as_root_ca_cert_path = as_root_ca_cert_path self.enclave_info_path = enclave_info_path def connect(self): + """Establish trusted connection and verify remote attestation report. + + Returns: + FrontendService: The original object which can be chained + with other methods. + """ sock = socket.create_connection(self.address) - channel = self.context.wrap_socket(sock, - server_hostname=self.address[0]) + channel = self._context.wrap_socket(sock, + server_hostname=self.address[0]) cert = channel.getpeercert(binary_form=True) - verify_report(self.as_root_ca_cert_path, self.enclave_info_path, cert, - "frontend") + _verify_report(self.as_root_ca_cert_path, self.enclave_info_path, cert, + "frontend") + + self._channel = channel + return self - return channel + def get_client(self): + """Get a client of frontend service to send RPC requests. + + Returns: + FrontendClient: Used for send/receive RPC requests. + """ + return FrontendClient(self._channel) class RegisterFunctionRequest: - def __init__(self, metadata, name, description, executor_type, public, - payload, arguments, inputs, outputs): + def __init__(self, metadata: Metadata, name: str, description: str, + executor_type: str, public: bool, payload: List[int], + arguments: List[str], inputs: List[FunctionInput], + outputs: List[FunctionOutput]): self.request = "register_function" self.metadata = metadata self.name = name @@ -106,7 +257,8 @@ def __init__(self, metadata, name, description, executor_type, public, class RegisterInputFileRequest: - def __init__(self, metadata, url, cmac, crypto_info): + def __init__(self, metadata: Metadata, url: str, cmac: str, + crypto_info: CryptoInfo): self.request = "register_input_file" self.metadata = metadata self.url = url @@ -115,30 +267,34 @@ def __init__(self, metadata, url, cmac, crypto_info): class RegisterOutputFileRequest: - def __init__(self, metadata, url, crypto_info): + def __init__(self, metadata: Metadata, url: str, crypto_info: CryptoInfo): self.request = "register_output_file" self.metadata = metadata self.url = url self.crypto_info = crypto_info + class UpdateInputFileRequest: - def __init__(self, metadata, data_id, url): + def __init__(self, metadata: Metadata, data_id: str, url: str): self.request = "update_input_file" self.metadata = metadata - self.data_id =data_id + self.data_id = data_id self.url = url class UpdateOutputFileRequest: - def __init__(self, metadata, data_id, url): + def __init__(self, metadata: Metadata, data_id: str, url: str): self.request = "update_output_file" self.metadata = metadata - self.data_id =data_id + self.data_id = data_id self.url = url + class CreateTaskRequest: - def __init__(self, metadata, function_id, function_arguments, executor, - inputs_ownership, outputs_ownership): + def __init__(self, metadata: Metadata, function_id: str, + function_arguments: Dict[str, Any], executor: str, + inputs_ownership: List[OwnerList], + outputs_ownership: List[OwnerList]): self.request = "create_task" self.metadata = metadata self.function_id = function_id @@ -149,7 +305,8 @@ def __init__(self, metadata, function_id, function_arguments, executor, class AssignDataRequest: - def __init__(self, metadata, task_id, inputs, outputs): + def __init__(self, metadata: Metadata, task_id: str, inputs: List[DataMap], + outputs: List[DataMap]): self.request = "assign_data" self.metadata = metadata self.task_id = task_id @@ -158,106 +315,102 @@ def __init__(self, metadata, task_id, inputs, outputs): class ApproveTaskRequest: - def __init__(self, metadata, task_id): + def __init__(self, metadata: Metadata, task_id: str): self.request = "approve_task" self.metadata = metadata self.task_id = task_id class InvokeTaskRequest: - def __init__(self, metadata, task_id): + def __init__(self, metadata: Metadata, task_id: str): self.request = "invoke_task" self.metadata = metadata self.task_id = task_id class GetTaskRequest: - def __init__(self, metadata, task_id): + def __init__(self, metadata: Metadata, task_id: str): self.request = "get_task" self.metadata = metadata self.task_id = task_id -class TeaclaveFile128Key: - def __init__(self, schema, key, iv): - self.schema = schema - self.key = key - self.iv = iv - - class FrontendClient: - def __init__(self, channel, metadata): + def __init__(self, channel: ssl.SSLSocket, metadata: Metadata = None): self.channel = channel self.metadata = metadata def register_function(self, - name, - description, - executor_type, - public=True, - payload=[], - arguments=[], - inputs=[], - outputs=[]): + name: str, + description: str, + executor_type: str, + public: bool = True, + payload: List[int] = [], + arguments: List[str] = [], + inputs: List[FunctionInput] = [], + outputs: List[FunctionOutput] = []): request = RegisterFunctionRequest(self.metadata, name, description, executor_type, public, payload, arguments, inputs, outputs) - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + response = _read_message(self.channel) return response["content"]["function_id"] - def register_input_file(self, url, schema, key, iv, cmac): + def register_input_file(self, url: str, schema: str, key: List[int], + iv: List[int], cmac: str): request = RegisterInputFileRequest(self.metadata, url, cmac, - TeaclaveFile128Key(schema, key, iv)) - write_message(self.channel, request) - response = read_message(self.channel) + CryptoInfo(schema, key, iv)) + _write_message(self.channel, request) + response = _read_message(self.channel) return response["content"]["data_id"] - def register_output_file(self, url, schema, key, iv): - request = RegisterOutputFileRequest( - self.metadata, url, TeaclaveFile128Key(schema, key, iv)) - write_message(self.channel, request) - response = read_message(self.channel) + def register_output_file(self, url: str, schema: str, key: List[int], + iv: List[int]): + request = RegisterOutputFileRequest(self.metadata, url, + CryptoInfo(schema, key, iv)) + _write_message(self.channel, request) + response = _read_message(self.channel) return response["content"]["data_id"] def create_task(self, - function_id, - function_arguments, - executor, - inputs_ownership=[], - outputs_ownership=[]): + function_id: str, + function_arguments: Dict[str, Any], + executor: str, + inputs_ownership: List[OwnerList] = [], + outputs_ownership: List[OwnerList] = []): function_arguments = json.dumps(function_arguments) request = CreateTaskRequest(self.metadata, function_id, function_arguments, executor, inputs_ownership, outputs_ownership) - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + response = _read_message(self.channel) return response["content"]["task_id"] - def assign_data_to_task(self, task_id, inputs, outputs): + def assign_data_to_task(self, task_id: str, inputs: List[DataMap], + outputs: List[DataMap]): request = AssignDataRequest(self.metadata, task_id, inputs, outputs) - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + _ = _read_message(self.channel) return - def approve_task(self, task_id): + def approve_task(self, task_id: str): request = ApproveTaskRequest(self.metadata, task_id) - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + _ = _read_message(self.channel) return - def invoke_task(self, task_id): + def invoke_task(self, task_id: str): request = InvokeTaskRequest(self.metadata, task_id) - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + response = _read_message(self.channel) assert (response["result"] == "ok") - def get_task_result(self, task_id): + def get_task_result(self, task_id: str): request = GetTaskRequest(self.metadata, task_id) while True: - write_message(self.channel, request) - response = read_message(self.channel) + _write_message(self.channel, request) + response = _read_message(self.channel) time.sleep(1) if response["content"]["status"] == 10: break @@ -265,21 +418,25 @@ def get_task_result(self, task_id): return response["content"]["result"]["result"]["Ok"]["return_value"] -def write_message(sock, message): +def _write_message(sock: ssl.SSLSocket, message: Any): + class RequestEncoder(json.JSONEncoder): + def default(self, o): + return o.__dict__ + message = json.dumps(message, cls=RequestEncoder).encode() sock.write(struct.pack(">Q", len(message))) sock.write(message) -def read_message(sock): +def _read_message(sock: ssl.SSLSocket): response_len = struct.unpack(">Q", sock.read(8)) response = sock.read(response_len[0]) response = json.loads(response) return response -def verify_report(as_root_ca_cert_path, enclave_info_path, cert, - endpoint_name): +def _verify_report(as_root_ca_cert_path: str, enclave_info_path: str, + cert: Dict[str, Any], endpoint_name: str): if os.environ.get('SGX_MODE') == 'SW': return diff --git a/services/README.md b/services/README.md index f00a73220..ad6c6ca62 100644 --- a/services/README.md +++ b/services/README.md @@ -38,6 +38,24 @@ a safe and secure FaaS platform. scheduler service to complete tasks. There could be many execution service instances (or nodes) with different capabilities deployed in a cloud infrastructure. + +## Structure + +A service is consist of two parts: app (untrusted) and enclave (trusted). The +app part is responsible for launching and terminating the service, which the +enclave part is to serve RPC requests from trusted channels. Typically, a service's +implementation contains two important structs and one trait. Let's take the +frontend service as an example. + +- `TeaclaveFrontendService` (struct): Define properties or configurations along + with the lifetime of the service. For example, the frontend service need to + hold clients (with established trusted channels) to communicate with the + authentication service and management service. +- `TeaclaveFrontendError` (struct): Define errors which may occur in this + service, authentication error, for example. +- `TeaclaveFrontend` (trait): Define functions (requests) the service need to + handle. The trait will be automatically derived from definitions in the + ProtoBuf file and can be imported from the `teaclave_proto` crate. ## RPC and Protocols diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.enc b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.enc new file mode 100644 index 000000000..58001543b Binary files /dev/null and b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.enc differ diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.txt b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.txt new file mode 100644 index 000000000..1816421a3 --- /dev/null +++ b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_a.txt @@ -0,0 +1,5 @@ +b : 2000 +a : 100 +c : 30000 +e : 5000000 +d : 400000 diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.enc b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.enc new file mode 100644 index 000000000..eb76a7db1 Binary files /dev/null and b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.enc differ diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.txt b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.txt new file mode 100644 index 000000000..06b09c23f --- /dev/null +++ b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_b.txt @@ -0,0 +1,5 @@ +e : 3000 +x : 200 +c : 40000 +y : 10 +a : 5000000 \ No newline at end of file diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.enc b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.enc new file mode 100644 index 000000000..5fa5ec1cf Binary files /dev/null and b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.enc differ diff --git a/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.txt b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.txt new file mode 100644 index 000000000..7233cbc81 --- /dev/null +++ b/tests/fixtures/functions/private_join_and_compute/three_party_data/bank_c.txt @@ -0,0 +1,5 @@ +e : 30000 +x : 200 +c : 400000 +y : 10 +d : 5000000