Aryeh/rpc-auth

.gitignore

@@ -4,4 +4,9 @@ dist
 *.pyc
 *.egg-info
 .venv*
+.DS_Store
 build
+.devspace/
+mkchain-devspace.yaml
+/*_chain.yaml
+/*_chain_invite.yaml

DEVELOPMENT.md

@@ -18,3 +18,26 @@ Build all containers:
 devspace build --skip-push --tag=dev
 ```
 
+# Develop with devspace
+(This is all still being worked on! Things should become more clear with the introduction of Helm.)
+
+## Deploy a chain and have devspace watch only k8s manifest:
+- Generate constants (with or without Zerotier flags)
+- `devspace dev --var=CHAIN_NAME="$CHAIN_NAME"`
+
+## Deploy chain with Zerotier and also watch the ZT docker file:
+- Generate constants with ZT flags
+- Use zerotier profile: `devspace dev --var=CHAIN_NAME="$CHAIN_NAME" -p zerotier`
+- You may leave out the profile flag if you don't want devspace to watch for zerotier file changes.
+
+## RPC Auth backend
+I've experimented with the idea of giving each service its own devspace.yaml. Inside RPC auth's devspace.yaml, it defines mkchain as a dependency which by running RPC auth devspace.yaml as the entrypoint, will also spin up mkchain (with zerotier if you generated it). Devspace is currently limited in that it will not reload dependencies if their files change. Therefore right now, the recommended way if you would like to view logs and/or have dependencies auto reload, is to run `devspace dev/logs` in multiple shells.
+
+### Deploy RPC auth backend and watch its files:
+- `cd ./docker/rpc-auth`
+- `devspace dev --var=CHAIN_NAME="$CHAIN_NAME"`
+
+## Misc
+- Devspace runs a hook to increase `fs.inotify.max_user_watches` to 1048576 in minikube. This is to avoid a "no space left on device" error. See [here](https://serverfault.com/questions/963529/minikube-k8s-kubectl-failed-to-watch-file-no-space-left-on-device) for more.
+- If you would like to avoid passing `--var=CHAIN_NAME="$CHAIN_NAME"` to devspace, you can execute in your shell `export CHAIN_NAME=my_chain`.
+- You can pass mkchain args like so: `devspace dev --var=CHAIN_NAME="$CHAIN_NAME" --var=MKCHAIN_ARGS="--number-of-nodes 2"`

MULTICLUSTER.md

@@ -14,7 +14,7 @@ peer-to-peer network. This tutorial assumes the use of Minikube.
 * a ZeroTier network and api access token
 * jq
 
-## Installing prerequisites 
+## Installing prerequisites
 
 This section varies depending on OS.
 
@@ -34,7 +34,7 @@ brew install jq
 brew install minikube
 ```
 
-### Arch Linux 
+### Arch Linux
 
 ```shell
 pacman -Syu && pacman -S python3 jq minikube kubectl kubectx linux
@@ -162,3 +162,26 @@ Check that the nodes have matching heads by comparing their hashes:
 ``` shell
 kubectl get pod -n tqtezos -l appType=tezos -o name | while read line; do kubectl -n tqtezos exec $line -c tezos-node -- /usr/local/bin/tezos-client rpc get /chains/main/blocks/head/hash; done
 ```
+
+## RPC Authentication
+You can optionally spin up an RPC authentication server allowing clients with your given permission to make RPC calls:
+
+```shell
+mkchain create $CHAIN_NAME --rpc-auth ...
+```
+
+### Current authentication flow
+The client authenticates themselves and will receive a secret url that allows them to make RPC calls.
+- You provide a trusted client with your cluster ip/address and your private tezos chain id.
+- To see your chain id:
+  ```shell
+  kubectl exec -it -n tqtezos deployment/tezos-bootstrap-node -c tezos-node -- tezos-client rpc get /chains/main/chain_id
+  ```
+  Or use a tool like like [Lens](https://k8slens.dev/) or run `kubectl logs -n tqtezos deployment/tezos-bootstrap-node -c tezos-node` to see the tezos node logs at the top.
+- client runs: `scripts/rpc-auth-client.sh --cluster-address $CLUSTER_IP --tz-alias $TZ_ALIAS --chain-id $CHAIN_ID`
+- TZ_ALIAS is the alias of a client's tz address secret key. The client's secret key is used to sign some data for the server to then verify.
+- If the client is authenticated, the response should contain a secret url such as `http://192.168.64.51/tezos-node-rpc/ffff3eb3d7dd4f6bbff3f2fd096722ae/`
+- Client can then make RPC requests:
+  - `curl http://192.168.64.51/tezos-node-rpc/ffff3eb3d7dd4f6bbff3f2fd096722ae/chains/main/chain_id`
+  - Bug in tezos client v8, so as of version `tezos/tezos:master_08d3405e_20201113152010`:
+    - `tezos-client --endpoint http://192.168.64.51/tezos-node-rpc/ffff3eb3d7dd4f6bbff3f2fd096722ae/ rpc get chains/main/chain_id`

README.md

@@ -44,6 +44,7 @@ You can modify these parameters by:
 | timestamp | --timestamp | timestamp for the chain to join | |
 | protocol_hash | --protocol-hash | Desired Tezos protocol hash | PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo |
 | baker_command | --baker-command | The baker command to use, including protocol | tezos-baker-007-PsDELPH1 |
+| rpc_auth | --rpc-auth | Include an RPC authentication server | |
 
 ## private chain
 

devspace.yaml

@@ -5,31 +5,55 @@ deployments:
     kubectl:
       manifests:
         - ./mkchain-devspace.yaml
-images:
-  zerotier:
-    image: tezos-zerotier
-    dockerfile: ./docker/zerotier/Dockerfile
-    context: ./docker/zerotier
+
 dev:
-  sync:
-    - imageName: zerotier
-      namespace: tqtezos
-      labelSelector:
-        app: tezos-bootstrap-node
+  logs:
+    disabled: true
+  autoReload:
+    deployments:
+      - chain
+    paths:
+      - ./tqchain/deployment/*
+
 hooks:
   - command: sh
     args:
       - -c
-      - "mkchain ${ACTION} ${CHAIN_NAME} > mkchain-devspace.yaml"
+      -  minikube ssh "sudo sysctl fs.inotify.max_user_watches=1048576"
     when:
       before:
         deployments: all
+  - command: sh
+    args:
+      - -c
+      - "mkchain ${ACTION} ${CHAIN_NAME} ${CHAIN_ARGS} > mkchain-devspace.yaml"
+    when:
+      before:
+        deployments: all
+
 vars:
-- name: CHAIN_NAME
-  question: Name of the chain as passed to generate-constants
-  default: "devspace"
-  source: env
-- name: ACTION
-  question: Mkchain action (create or invite)
-  default: "create"
-  source: env
+  - name: ACTION
+    question: Mkchain action (create or invite)
+    default: "create"
+    source: env
+  - name: CHAIN_NAME
+    question: Name of the chain as passed to generate-constants
+    default: "devspace"
+    source: env
+  - name: CHAIN_ARGS
+    default: ""
+    source: env
+
+profiles:
+  - name: zerotier
+    replace:
+      images:
+        zerotier:
+          image: tezos-zerotier
+          dockerfile: ./docker/zerotier/Dockerfile
+          context: ./docker/zerotier
+    patches:
+      - op: add
+        path: dev.autoReload.images
+        value:
+          - zerotier

docker/rpc-auth/Dockerfile

@@ -0,0 +1,66 @@
+FROM python:3.9-slim as builder
+
+RUN mkdir -p /var/rpc-auth/
+
+WORKDIR /var/rpc-auth/
+
+# Installing pytezos deps
+RUN apt-get update -y \
+  && apt-get install --no-install-recommends -y \
+  automake \
+  build-essential \
+  libffi-dev \
+  libgmp-dev \
+  libsecp256k1-dev \
+  libsodium-dev \
+  libtool \
+  pkg-config \
+  && echo
+
+COPY requirements.txt .
+RUN mkdir wheels \
+  && pip wheel -r requirements.txt \
+  --wheel-dir ./wheels --no-cache-dir
+
+FROM python:3.9-slim AS src
+
+WORKDIR /var/rpc-auth/
+
+# Installing pytezos deps
+RUN apt-get update -y \
+  && apt-get install --no-install-recommends -y \
+  libffi-dev \
+  libgmp-dev \
+  libsecp256k1-dev \
+  libsodium-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+# Installing python dependencies
+COPY --from=builder /var/rpc-auth/wheels wheels
+COPY requirements.txt .
+RUN pip install -r requirements.txt \
+  --no-index --find-links ./wheels \
+  && rm -rf ./wheels ./requirements.txt
+
+RUN groupadd -g 999 appuser && \
+  useradd -r -u 999 -g appuser appuser
+
+COPY --chown=appuser:appuser ./server/index.py .
+
+ARG FLASK_ENV=production
+ENV FLASK_ENV=$FLASK_ENV
+ENV PYTHONUNBUFFERED=x
+
+EXPOSE 8080
+
+USER appuser
+
+CMD uwsgi \
+  --http-socket 0.0.0.0:8080 \
+  --callable app \
+  --threads 100 \
+  --processes 1 \
+  --wsgi-file index.py \
+  --worker-reload-mercy 0 \
+  $(if [ "${FLASK_ENV}" = "development" ] ; then echo "--touch-reload index.py" ; else : ; fi)

docker/rpc-auth/devspace.yaml

@@ -0,0 +1,54 @@
+version: v1beta9
+deployments:
+  - name: rpc-auth
+    namespace: tqtezos
+    kubectl:
+      manifests:
+        - ../../tqchain/deployment/rpc-auth.yaml
+
+images:
+  rpc-auth:
+    image: tezos-rpc-auth
+    dockerfile: ./Dockerfile
+    context: ./
+    build:
+      docker:
+        options:
+          buildArgs:
+            FLASK_ENV: ${FLASK_ENV}
+dev:
+  logs:
+    images:
+      - rpc-auth
+  sync:
+    - imageName: rpc-auth
+      namespace: tqtezos
+      labelSelector:
+        app: rpc-auth
+      localSubPath: ./server
+  autoReload:
+    images:
+      - rpc-auth
+    deployments:
+      - rpc-auth
+
+hooks:
+  - command: minikube
+    args:
+      - addons
+      - enable
+      - ingress
+    when:
+      before:
+        deployments: rpc-auth
+
+vars:
+  - name: FLASK_ENV # development | production
+    question: Deployment env to run RPC authentication
+    default: development
+    source: env
+
+dependencies:
+  - name: chain
+    source:
+      path: ../../

docker/rpc-auth/requirements.txt

@@ -0,0 +1,35 @@
+base58==1.0.3
+bson==0.5.10
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+click==7.1.2
+fastecdsa==1.7.5
+fire==0.3.1
+Flask==1.1.2
+idna==2.10
+itsdangerous==1.1.0
+Jinja2==2.11.2
+loguru==0.5.3
+MarkupSafe==1.1.1
+mnemonic==0.19
+netstruct==1.1.2
+pendulum==2.1.2
+ply==3.11
+pyblake2==1.1.2
+pycparser==2.20
+pysodium==0.7.5
+pytezos==2.5.11
+python-dateutil==2.8.1
+pytzdata==2020.1
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+secp256k1==0.13.2
+simplejson==3.17.2
+six==1.15.0
+termcolor==1.1.0
+tqdm==4.51.0
+urllib3==1.25.11
+uWSGI==2.0.19.1
+Werkzeug==1.0.1