From 733eceef4af4b2a41bd57463fb5de7122916aaee Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Tue, 16 Feb 2021 20:51:48 -0800 Subject: [PATCH 01/17] support for public networks * generate config is now much simpler, gets rid of argparse * replace zerotier in use true/false with network_type public/private/isolated --- charts/tezos/templates/baker.yaml | 4 +- charts/tezos/templates/configs.yaml | 3 +- charts/tezos/templates/node.yaml | 4 +- charts/tezos/values.yaml | 3 +- config-generator/entrypoint.py | 129 +++++++--------------------- 5 files changed, 37 insertions(+), 106 deletions(-) diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index 8fd5692a3..b4a8c0e3e 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -138,7 +138,7 @@ spec: fieldPath: metadata.name - name: DAEMON value: endorser -{{- if .Values.zerotier_in_use }} +{{- if eq .Values.chain_type "private" }} - args: - "-c" - "echo 'starting zerotier' && zerotier-one /var/tezos/zerotier" @@ -189,7 +189,7 @@ spec: volumeMounts: - mountPath: /var/tezos name: var-volume -{{- if .Values.zerotier_in_use }} +{{- if eq .Values.chain_type "private" }} - envFrom: - configMapRef: name: zerotier-config diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index b61545a21..f45f84e9d 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -9,7 +9,8 @@ data: "genesis_block": "{{ .Values.genesis.genesis_chain_id }}", "activation_account": "{{ (index .Values.accounts 0).name }}", "timestamp": "{{ .Values.genesis.bootstrap_timestamp }}", - "zerotier_in_use": {{ .Values.zerotier_in_use }}, + "chain_type": "{{ .Values.chain_type }}", + "network": "{{ .Values.network }}", "proto_command": "{{ .Values.protocol.command }}", "protocol_hash": "{{ .Values.protocol.hash }}" } diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml index fd158f7cf..8b9e942df 100644 --- a/charts/tezos/templates/node.yaml +++ b/charts/tezos/templates/node.yaml @@ -36,7 +36,7 @@ spec: name: config-volume - mountPath: /var/tezos name: var-volume -{{- if .Values.zerotier_in_use }} +{{- if eq .Values.chain_type "private" }} - args: - "-c" - "echo 'starting zerotier' && zerotier-one /var/tezos/zerotier" @@ -93,7 +93,7 @@ spec: volumeMounts: - mountPath: /var/tezos name: var-volume -{{- if .Values.zerotier_in_use }} +{{- if eq .Values.chain_type "private" }} - envFrom: - configMapRef: name: zerotier-config diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 49c196013..0fa2dd5dd 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -2,7 +2,8 @@ baker: true chain_name: YOUR_CHAIN_NAME_HERE is_invitation: false rpc_auth: false -zerotier_in_use: false +chain_type: public +network: edo2net zerotier_config: zerotier_network: YOUR_NETWORK_ID_HERE zerotier_token: YOUR_ZEROTIER_TOKEN_HERE diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index 882c915d6..c182d95ce 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -36,7 +36,7 @@ def main(): if main_args.generate_config_json: net_addr = None bootstrap_peers = CHAIN_PARAMS.get("bootstrap_peers", []) - if CHAIN_PARAMS["zerotier_in_use"]: + if CHAIN_PARAMS["chain_type"] == "private": with open("/var/tezos/zerotier_data.json", "r") as f: net_addr = json.load(f)[0]["assignedAddresses"][0].split("/")[0] if bootstrap_peers == []: @@ -99,105 +99,34 @@ def get_node_config( net_addr=None, ): - p2p = ["p2p"] - for bootstrap_peer in bootstrap_peers: - p2p.extend(["--bootstrap-peers", bootstrap_peer]) - if net_addr: - p2p.extend(["--listen-addr", net_addr + ":9732"]) - - node_config_args = p2p + [ - "global", - "rpc", - "network", - "--chain-name", - chain_name, - "genesis", - "--timestamp", - timestamp, - "--block", - genesis_block, - "genesis_parameters", - "--genesis-pubkey", - genesis_key, - ] - - return generate_node_config(node_config_args) - - -# FIXME - this should probably be replaced with subprocess calls to tezos-node-config -def generate_node_config(node_argv): - parser = argparse.ArgumentParser(prog="nodeconfig") - subparsers = parser.add_subparsers(help="sub-command help", dest="subparser_name") - - global_parser = subparsers.add_parser("global") - global_parser.add_argument("--data-dir", default="/var/tezos/node") - - rpc_parser = subparsers.add_parser("rpc") - rpc_parser.add_argument( - "--listen-addrs", - action="append", - default=[f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], - ) - - p2p_parser = subparsers.add_parser("p2p") - p2p_parser.add_argument("--bootstrap-peers", action="append", default=[]) - p2p_parser.add_argument("--listen-addr", default="[::]:9732") - p2p_parser.add_argument("--expected-proof-of-work", default=0, type=int) - - network_parser = subparsers.add_parser("network") - network_parser.add_argument("--chain-name") - network_parser.add_argument("--sandboxed-chain-name", default="SANDBOXED_TEZOS") - network_parser.add_argument( - "--default-bootstrap-peers", action="append", default=[] - ) - - genesis_parser = subparsers.add_parser("genesis") - genesis_parser.add_argument("--timestamp") - genesis_parser.add_argument( - "--block", default="BLockGenesisGenesisGenesisGenesisGenesisd6f5afWyME7" - ) - genesis_parser.add_argument( - "--protocol", default="PtYuensgYBb3G3x1hLLbCmcav8ue8Kyd2khADcL5LsT5R1hcXex" - ) - - genesis_parameters_parser = subparsers.add_parser("genesis_parameters") - genesis_parameters_parser.add_argument("--genesis-pubkey") - - namespaces = [] - while node_argv: - namespace, node_argv = parser.parse_known_args(node_argv) - namespaces.append(namespace) - if not namespace.subparser_name: - break - - node_config = {} - special_keys = [ - "listen_addrs", - "bootstrap_peers", - "data_dir", - "listen_addr", - "expected_proof_of_work", - ] - for namespace in namespaces: - section = vars(namespace) - fixed_section = {} - for k, v in section.items(): - if k in special_keys: - fixed_section[k.replace("_", "-")] = v - else: - fixed_section[k] = v - - key = fixed_section.pop("subparser_name") - if key == "global": - node_config.update(fixed_section) - else: - # doubly nested parsers are a bit tricky. we'll just force the network keys where they belong - if key == "genesis": - node_config["network"][key] = fixed_section - elif key == "genesis_parameters": - node_config["network"][key] = {"values": fixed_section} - else: - node_config[key] = fixed_section + node_config = { "data-dir": "/var/tezos/node", + "rpc": { + "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], + }, + "p2p": { + "expected-proof-of-work": 0, + "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" ) + } + } + if CHAIN_PARAMS["chain_type"] == "public": + node_config["network"] = CHAIN_PARAMS["network"] + else: + node_config["p2p"]["bootstrap-peers"] = bootstrap_nodes + node_config["network"] = { + "chain_name": chain_name, + "sandboxed_chain_name": "SANDBOZED_TEZOS", + "default_bootstrap_peers": [], + "genesis": { + "timestamp": timestamp, + "block": genesis_block, + "protocol": "PtYuensgYBb3G3x1hLLbCmcav8ue8Kyd2khADcL5LsT5R1hcXex" + }, + "genesis_parameters": { + "values": { + "genesis_pubkey": genesis_key, + }, + }, + } return node_config From 24f724fbc7c6038c2e88361d9aa5dbaf54bd192d Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 22 Feb 2021 16:17:33 -0800 Subject: [PATCH 02/17] run tezos-node config init and extract bootstrap ppers Done as part of config-generator. It is now using the tezos container as base. remove identity generation - nodes does it temporarily change tezos container version to master - will revert --- baker-endorser/Dockerfile | 2 +- chain-initiator/Dockerfile | 2 +- charts/tezos/templates/baker.yaml | 15 +-------------- config-generator/Dockerfile | 10 +++++----- config-generator/entrypoint.py | 12 ++++++++---- config-generator/entrypoint.sh | 26 ++++++++++++++++++++++++++ wait-for-bootstrap/Dockerfile | 2 +- 7 files changed, 43 insertions(+), 26 deletions(-) create mode 100755 config-generator/entrypoint.sh diff --git a/baker-endorser/Dockerfile b/baker-endorser/Dockerfile index 38325f207..987042d7c 100644 --- a/baker-endorser/Dockerfile +++ b/baker-endorser/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:v8-release +FROM tezos/tezos:master RUN sudo apk add jq bash COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/chain-initiator/Dockerfile b/chain-initiator/Dockerfile index 05c0cbd09..c5b8d2e25 100644 --- a/chain-initiator/Dockerfile +++ b/chain-initiator/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:v8-release +FROM tezos/tezos:master RUN sudo apk add jq COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index 4ff8df6ad..e5c6d206c 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.is_invitation }} +{{- if and (not .Values.is_invitation) (not (eq .Values.chain_type "public")) }} apiVersion: batch/v1 kind: Job metadata: @@ -150,19 +150,6 @@ spec: name: var-volume {{- end }} initContainers: - - args: - - "-c" - - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)" - command: - - /bin/sh - image: {{ .Values.images.tezos | quote }} - name: identity-job - imagePullPolicy: IfNotPresent - volumeMounts: - - mountPath: /etc/tezos - name: config-volume - - mountPath: /var/tezos - name: var-volume {{- if not .Values.is_invitation }} - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} imagePullPolicy: IfNotPresent diff --git a/config-generator/Dockerfile b/config-generator/Dockerfile index 7f94bad82..73930922f 100644 --- a/config-generator/Dockerfile +++ b/config-generator/Dockerfile @@ -1,8 +1,8 @@ -FROM python:alpine +FROM tezos/tezos:master ENV PYTHONUNBUFFERED=1 -RUN apk add --no-cache --virtual .build-deps gcc musl-dev libffi-dev make \ - && pip install base58 pynacl \ - && apk del .build-deps gcc musl-dev libffi-dev make +RUN sudo apk add --no-cache --virtual .build-deps jq gcc py3-pip python3-dev musl-dev libffi-dev make \ + && pip install base58 pynacl COPY entrypoint.py / -ENTRYPOINT ["python", "/entrypoint.py"] +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] CMD [] diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index 3587a9747..62a4a4451 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -48,6 +48,9 @@ def main(): net_addr = json.load(f)[0]["assignedAddresses"][0].split("/")[0] if bootstrap_peers == []: bootstrap_peers.extend(get_zerotier_bootstrap_peer_ips()) + if CHAIN_PARAMS["chain_type"] == "public": + with open("/tmp/data/config.json", "r") as f: + bootstrap_peers.extend(json.load(f)["p2p"]["bootstrap-peers"]) else: local_bootstrap_peers = [] bakers = CHAIN_PARAMS["nodes"]["baking"] @@ -114,14 +117,15 @@ def get_node_config( "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], }, "p2p": { - "expected-proof-of-work": 0, + "bootstrap-peers": bootstrap_peers, "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" ) - } - } + }, + #"log": { "level": "debug"}, + } if CHAIN_PARAMS["chain_type"] == "public": node_config["network"] = CHAIN_PARAMS["network"] else: - node_config["p2p"]["bootstrap-peers"] = bootstrap_nodes + node_config["p2p"]["expected-proof-of-work"] = 0 node_config["network"] = { "chain_name": chain_name, "sandboxed_chain_name": "SANDBOZED_TEZOS", diff --git a/config-generator/entrypoint.sh b/config-generator/entrypoint.sh new file mode 100755 index 000000000..f3fe2e7e8 --- /dev/null +++ b/config-generator/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +set -ex + +tezos_network=$(echo $CHAIN_PARAMS | jq -r '.network') +chain_type=$(echo $CHAIN_PARAMS | jq -r '.chain_type') + +if [ "${chain_type}" == "public" ]; then + printf "Writing custom configuration for public node\n" + mkdir -p /tmp/data + + # We use this command to extract the data we need from the binary + # the python script below + /usr/local/bin/tezos-node config init \ + --config-file /tmp/data/config.json \ + --data-dir /tmp/data \ + --network $tezos_network + + cat /tmp/data/config.json + echo "" + +fi + +python3 /entrypoint.py $@ + +rm -r /tmp/data/ diff --git a/wait-for-bootstrap/Dockerfile b/wait-for-bootstrap/Dockerfile index 7bf21b0cc..6eebdd111 100644 --- a/wait-for-bootstrap/Dockerfile +++ b/wait-for-bootstrap/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:v8-release +FROM tezos/tezos:master RUN sudo apk add jq netcat-openbsd COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] From 5bc4c0f87f979035421c3167cfbc2f643102d513 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Tue, 23 Feb 2021 23:54:05 -0800 Subject: [PATCH 03/17] add snapshot downloader change ordering of init containers again to make sure they all download snapshots --- baker-endorser/Dockerfile | 2 +- charts/tezos/templates/baker.yaml | 69 +++++++++++++---------------- charts/tezos/templates/configs.yaml | 1 + charts/tezos/templates/node.yaml | 52 +++++++++++----------- charts/tezos/values.yaml | 2 + config-generator/entrypoint.py | 5 ++- devspace.yaml | 4 ++ snapshot-downloader/Dockerfile | 5 +++ snapshot-downloader/entrypoint.sh | 31 +++++++++++++ 9 files changed, 104 insertions(+), 67 deletions(-) create mode 100644 snapshot-downloader/Dockerfile create mode 100755 snapshot-downloader/entrypoint.sh diff --git a/baker-endorser/Dockerfile b/baker-endorser/Dockerfile index 987042d7c..38325f207 100644 --- a/baker-endorser/Dockerfile +++ b/baker-endorser/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master +FROM tezos/tezos:v8-release RUN sudo apk add jq bash COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index e5c6d206c..6c683a6c3 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -93,24 +93,6 @@ spec: name: config-volume - mountPath: /var/tezos name: var-volume - - image: "{{ .Values.tezos_k8s_images.baker_endorser }}" - imagePullPolicy: IfNotPresent - name: baker - volumeMounts: - - mountPath: /var/tezos - name: var-volume - envFrom: - - configMapRef: - name: tezos-config - - secretRef: - name: tezos-secret - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: DAEMON - value: baker - image: "{{ .Values.tezos_k8s_images.baker_endorser }}" imagePullPolicy: IfNotPresent name: endorser @@ -150,6 +132,37 @@ spec: name: var-volume {{- end }} initContainers: + - image: {{ .Values.tezos_k8s_images.config_generator }} + imagePullPolicy: IfNotPresent + name: config-generator + args: + - "--generate-config-json" + envFrom: + - secretRef: + name: tezos-secret + - configMapRef: + name: tezos-config + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /etc/tezos + name: config-volume + - mountPath: /var/tezos + name: var-volume + - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}" + imagePullPolicy: IfNotPresent + name: snapshot-downloader + volumeMounts: + - mountPath: /var/tezos + name: var-volume + - mountPath: /etc/tezos + name: config-volume + envFrom: + - configMapRef: + name: tezos-config {{- if not .Values.is_invitation }} - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} imagePullPolicy: IfNotPresent @@ -183,26 +196,6 @@ spec: fieldRef: fieldPath: metadata.name {{- end }} - - image: {{ .Values.tezos_k8s_images.config_generator }} - imagePullPolicy: IfNotPresent - name: config-generator - args: - - "--generate-config-json" - envFrom: - - secretRef: - name: tezos-secret - - configMapRef: - name: tezos-config - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/tezos - name: config-volume - - mountPath: /var/tezos - name: var-volume securityContext: fsGroup: 100 volumes: diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index 47c0b7059..0587db634 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -11,6 +11,7 @@ data: "timestamp": "{{ .Values.genesis.timestamp }}", "chain_type": "{{ .Values.chain_type }}", "network": "{{ .Values.network }}", + "snapshot_url": "{{ .Values.snapshot_url }}", "proto_command": "{{ .Values.protocol.command }}", "protocol_hash": "{{ .Values.protocol.hash }}" } diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml index 49aa72979..55366b37f 100644 --- a/charts/tezos/templates/node.yaml +++ b/charts/tezos/templates/node.yaml @@ -63,19 +63,37 @@ spec: name: var-volume {{- end }} initContainers: - - args: - - "-c" - - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)" - command: - - /bin/sh - image: {{ .Values.images.tezos | quote }} - name: identity-job + - image: {{ .Values.tezos_k8s_images.config_generator }} imagePullPolicy: IfNotPresent + name: config-generator + args: + - "--generate-config-json" + envFrom: + - secretRef: + name: tezos-secret + - configMapRef: + name: tezos-config + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP volumeMounts: - mountPath: /etc/tezos name: config-volume - mountPath: /var/tezos name: var-volume + - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}" + imagePullPolicy: IfNotPresent + name: snapshot-downloader + volumeMounts: + - mountPath: /var/tezos + name: var-volume + - mountPath: /etc/tezos + name: config-volume + envFrom: + - configMapRef: + name: tezos-config {{- if not .Values.is_invitation }} - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} imagePullPolicy: IfNotPresent @@ -109,26 +127,6 @@ spec: fieldRef: fieldPath: metadata.name {{- end }} - - image: {{ .Values.tezos_k8s_images.config_generator }} - imagePullPolicy: IfNotPresent - name: config-generator - args: - - "--generate-config-json" - envFrom: - - secretRef: - name: tezos-secret - - configMapRef: - name: tezos-config - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/tezos - name: config-volume - - mountPath: /var/tezos - name: var-volume securityContext: fsGroup: 100 volumes: diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 983183bb6..6610d2fd7 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -18,7 +18,9 @@ tezos_k8s_images: config_generator: tezos-k8s-config-generator:dev wait_for_bootstrap: tezos-k8s-wait-for-bootstrap:dev zerotier: tezos-k8s-zerotier:dev + snapshot_downloader: tezos-k8s-snapshot-downloader:dev +snapshot_url: https://mainnet.xtz-shots.io/rolling protocol: bootstrap_mutez: "4000000000000" hash: PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index 62a4a4451..f3107293b 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -112,7 +112,7 @@ def get_node_config( net_addr=None, ): - node_config = { "data-dir": "/var/tezos/node", + node_config = { "data-dir": "/var/tezos/node/data", "rpc": { "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], }, @@ -120,6 +120,9 @@ def get_node_config( "bootstrap-peers": bootstrap_peers, "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" ) }, + "shell": { + "history_mode": "rolling" + }, #"log": { "level": "debug"}, } if CHAIN_PARAMS["chain_type"] == "public": diff --git a/devspace.yaml b/devspace.yaml index 1b1d2c1df..74c9d3a5f 100755 --- a/devspace.yaml +++ b/devspace.yaml @@ -28,6 +28,10 @@ images: image: tezos-k8s-wait-for-bootstrap dockerfile: ./wait-for-bootstrap/Dockerfile context: ./wait-for-bootstrap + snapshot-downloader: + image: tezos-k8s-snapshot-downloader + dockerfile: ./snapshot-downloader/Dockerfile + context: ./snapshot-downloader config-generator: image: tezos-k8s-config-generator dockerfile: ./config-generator/Dockerfile diff --git a/snapshot-downloader/Dockerfile b/snapshot-downloader/Dockerfile new file mode 100644 index 000000000..5755235cc --- /dev/null +++ b/snapshot-downloader/Dockerfile @@ -0,0 +1,5 @@ +FROM tezos/tezos:master +RUN sudo apk add curl lz4 xz jq +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] +CMD [] diff --git a/snapshot-downloader/entrypoint.sh b/snapshot-downloader/entrypoint.sh new file mode 100755 index 000000000..0a53d1c9d --- /dev/null +++ b/snapshot-downloader/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -x + +bin_dir="/usr/local/bin" + +data_dir="/var/tezos" +node_dir="$data_dir/node" +node_data_dir="$node_dir/data" +node="$bin_dir/tezos-node" + +tezos_network=$(echo $CHAIN_PARAMS | jq -r '.network') +snapshot_url=$(echo $CHAIN_PARAMS | jq -r '.snapshot_url') +if [ -d ${node_dir}/data/context ]; then + echo "Blockchain has already been imported, exiting" + exit 0 +elif [ -z "$snapshot_url" ]; then + echo "No snapshot was passed as parameter, exiting" + exit 0 +else + echo "Did not find pre-existing data, importing blockchain" + mkdir -p ${node_dir}/data + echo '{ "version": "0.0.4" }' > ${node_dir}/version.json + cp -v /usr/local/share/tezos/alphanet_version ${node_dir} + snapshot_file=${node_dir}/chain.snapshot + curl -L -o $snapshot_file $snapshot_url + exec "${node}" snapshot import ${snapshot_file} --data-dir ${node_data_dir} --network $tezos_network --config-file /etc/tezos/config.json + find ${node_dir} + rm -rvf ${snapshot_file} + echo "" +fi From 874e07cca277fcba21d781eaf876d58baf9cae9c Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Fri, 26 Feb 2021 14:51:32 -0800 Subject: [PATCH 04/17] switch to tezos container forces us to use sudo --- config-generator/entrypoint.py | 4 +--- config-generator/entrypoint.sh | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index f3107293b..dc8d8effc 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -131,7 +131,7 @@ def get_node_config( node_config["p2p"]["expected-proof-of-work"] = 0 node_config["network"] = { "chain_name": chain_name, - "sandboxed_chain_name": "SANDBOZED_TEZOS", + "sandboxed_chain_name": "SANDBOXED_TEZOS", "default_bootstrap_peers": [], "genesis": { "timestamp": timestamp, @@ -279,8 +279,6 @@ def flatten_accounts(): def import_keys(): print("Importing keys") tezdir = "/var/tezos/client" - os.makedirs(tezdir, exist_ok=True) - os.chmod(tezdir, 0o777); secret_keys = [] public_keys = [] public_key_hashs = [] diff --git a/config-generator/entrypoint.sh b/config-generator/entrypoint.sh index f3fe2e7e8..67cc39471 100755 --- a/config-generator/entrypoint.sh +++ b/config-generator/entrypoint.sh @@ -21,6 +21,8 @@ if [ "${chain_type}" == "public" ]; then fi +sudo mkdir -p /var/tezos/client +sudo chmod -R 777 /var/tezos/client python3 /entrypoint.py $@ rm -r /tmp/data/ From e3201f1ea30a0151de2dfb0e655f156295f3065c Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Fri, 26 Feb 2021 14:53:40 -0800 Subject: [PATCH 05/17] I found a build which works well with edo2net; let's make it default --- baker-endorser/Dockerfile | 2 +- chain-initiator/Dockerfile | 2 +- charts/tezos/values.yaml | 2 +- config-generator/Dockerfile | 2 +- snapshot-downloader/Dockerfile | 2 +- wait-for-bootstrap/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/baker-endorser/Dockerfile b/baker-endorser/Dockerfile index 38325f207..0abb3ce22 100644 --- a/baker-endorser/Dockerfile +++ b/baker-endorser/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:v8-release +FROM tezos/tezos:master_328bc6c3_20210215194506 RUN sudo apk add jq bash COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/chain-initiator/Dockerfile b/chain-initiator/Dockerfile index c5b8d2e25..4c6e6ac4b 100644 --- a/chain-initiator/Dockerfile +++ b/chain-initiator/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master +FROM tezos/tezos:master_328bc6c3_20210215194506 RUN sudo apk add jq COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 6610d2fd7..9b8bdfc18 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -10,7 +10,7 @@ zerotier_config: # Images not part of the tezos-k8s repo go here images: - tezos: tezos/tezos:v8-release + tezos: master_328bc6c3_20210215194506 # Images that are part of the tezos-k8s repo go here tezos_k8s_images: baker_endorser: tezos-k8s-baker-endorser:dev diff --git a/config-generator/Dockerfile b/config-generator/Dockerfile index 73930922f..9a2492678 100644 --- a/config-generator/Dockerfile +++ b/config-generator/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master +FROM tezos/tezos:master_328bc6c3_20210215194506 ENV PYTHONUNBUFFERED=1 RUN sudo apk add --no-cache --virtual .build-deps jq gcc py3-pip python3-dev musl-dev libffi-dev make \ && pip install base58 pynacl diff --git a/snapshot-downloader/Dockerfile b/snapshot-downloader/Dockerfile index 5755235cc..caba59ad4 100644 --- a/snapshot-downloader/Dockerfile +++ b/snapshot-downloader/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master +FROM tezos/tezos:master_328bc6c3_20210215194506 RUN sudo apk add curl lz4 xz jq COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/wait-for-bootstrap/Dockerfile b/wait-for-bootstrap/Dockerfile index 6eebdd111..e4dc0ed67 100644 --- a/wait-for-bootstrap/Dockerfile +++ b/wait-for-bootstrap/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master +FROM tezos/tezos:master_328bc6c3_20210215194506 RUN sudo apk add jq netcat-openbsd COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] From 41d3cfaba82f227a7c0a40b45a2ada5b35f05252 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Fri, 26 Feb 2021 15:05:34 -0800 Subject: [PATCH 06/17] remove exec so the snapshot file is actually deleted --- snapshot-downloader/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshot-downloader/entrypoint.sh b/snapshot-downloader/entrypoint.sh index 0a53d1c9d..0fe1d2aeb 100755 --- a/snapshot-downloader/entrypoint.sh +++ b/snapshot-downloader/entrypoint.sh @@ -24,7 +24,7 @@ else cp -v /usr/local/share/tezos/alphanet_version ${node_dir} snapshot_file=${node_dir}/chain.snapshot curl -L -o $snapshot_file $snapshot_url - exec "${node}" snapshot import ${snapshot_file} --data-dir ${node_data_dir} --network $tezos_network --config-file /etc/tezos/config.json + ${node} snapshot import ${snapshot_file} --data-dir ${node_data_dir} --network $tezos_network --config-file /etc/tezos/config.json find ${node_dir} rm -rvf ${snapshot_file} echo "" From 80f62d0e2d732ae9449bbefe98532f0c0304562c Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Fri, 26 Feb 2021 16:04:10 -0800 Subject: [PATCH 07/17] re-add baker removed by mistake --- charts/tezos/templates/baker.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index 6c683a6c3..7e9847e67 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -93,6 +93,24 @@ spec: name: config-volume - mountPath: /var/tezos name: var-volume + - image: "{{ .Values.tezos_k8s_images.baker_endorser }}" + imagePullPolicy: IfNotPresent + name: baker + volumeMounts: + - mountPath: /var/tezos + name: var-volume + envFrom: + - configMapRef: + name: tezos-config + - secretRef: + name: tezos-secret + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: DAEMON + value: baker - image: "{{ .Values.tezos_k8s_images.baker_endorser }}" imagePullPolicy: IfNotPresent name: endorser From 776653bb7bd69110b3d5e0df8c3af8b15cb15ae3 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Fri, 26 Feb 2021 17:18:06 -0800 Subject: [PATCH 08/17] add mkchain support --- README.md | 14 ++++++++++++-- mkchain/tqchain/mkchain.py | 9 ++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8121c7b47..5617298a2 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,22 @@ Set [unbuffered IO](https://docs.python.org/3.6/using/cmdline.html#envvar-PYTHON export PYTHONUNBUFFERED=x ``` -## Start your chain +## Chain types + +There are 3 types of chains supported: + +* **isolated**: self-sufficient private chain. Does not connect to the outside world, +* **private**: private chain on a zerotier network. Can span multiple clusters, +* **public**: public chain such as a public testnet or Tezos mainnet. + +The `--chain-type` parameter lets you choose between these different types. + +## Start your private chain Run the following commands to create the Helm values, get the Helm chart repo, and install the Helm chart to start your chain. ```shell -mkchain $CHAIN_NAME --zerotier-network $ZT_NET --zerotier-token $ZT_TOKEN +mkchain $CHAIN_NAME --zerotier-network $ZT_NET --zerotier-token $ZT_TOKEN --chain-type private helm repo add tqtezos https://tqtezos.github.io/tezos-helm-charts diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index 5dd90260f..afee511ad 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -71,6 +71,10 @@ def get_genesis_vanity_chain_id(docker_image, seed_len=16): "default": 1, "type": int, }, + "chain_type": {"help": "Type of chain: public, private, isolated", + "default": "isolated", + }, + "network": {"help": "Name of the tezos public network to connect to "}, "zerotier_network": {"help": "Zerotier network id for external chain access"}, "zerotier_token": {"help": "Zerotier token for external chain access"}, "bootstrap_peer": {"help": "peer ip to join"}, @@ -156,7 +160,7 @@ def main(): "images": { "tezos": args.docker_image, }, - "zerotier_in_use": bool(args.zerotier_network), + "chain_type": args.chain_type, "rpc_auth": args.rpc_auth, "zerotier_config": { "zerotier_network": args.zerotier_network, @@ -190,6 +194,9 @@ def main(): "timestamp": datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(), } + if base_constants["chain_type"] == "public": + base_constants["network"] = ( args.network if args.network else "mainnet" ) + accounts = {"secret": [], "public": []} if old_create_values.get("accounts", None): accounts["secret"] = old_create_values["accounts"] From 5007325d5385a49cf81a705f51faf233304b1280 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sat, 27 Feb 2021 20:31:02 -0800 Subject: [PATCH 09/17] do not error when not public network --- config-generator/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config-generator/entrypoint.sh b/config-generator/entrypoint.sh index 67cc39471..873092986 100755 --- a/config-generator/entrypoint.sh +++ b/config-generator/entrypoint.sh @@ -25,4 +25,6 @@ sudo mkdir -p /var/tezos/client sudo chmod -R 777 /var/tezos/client python3 /entrypoint.py $@ -rm -r /tmp/data/ +if [ "${chain_type}" == "public" ]; then + rm -r /tmp/data/ +fi From d6e33effffffff29e700c4c7447ee389fdaedeea Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sat, 27 Feb 2021 20:55:13 -0800 Subject: [PATCH 10/17] move default protocol to edo2 move back zerotier ip before everything to fix zerotier use case make snapshot download only happen with public mode --- charts/tezos/templates/baker.yaml | 52 ++++++++++++++++--------------- charts/tezos/templates/node.yaml | 50 ++++++++++++++--------------- charts/tezos/values.yaml | 4 +-- 3 files changed, 54 insertions(+), 52 deletions(-) diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index 7e9847e67..d150bc686 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -150,6 +150,31 @@ spec: name: var-volume {{- end }} initContainers: +{{- if eq .Values.chain_type "private" }} + - envFrom: + - configMapRef: + name: zerotier-config + image: "{{ .Values.tezos_k8s_images.zerotier }}" + imagePullPolicy: IfNotPresent + name: get-zerotier-ip + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/tezos + name: var-volume + - mountPath: /dev/net/tun + name: dev-net-tun + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{- end }} - image: {{ .Values.tezos_k8s_images.config_generator }} imagePullPolicy: IfNotPresent name: config-generator @@ -170,6 +195,7 @@ spec: name: config-volume - mountPath: /var/tezos name: var-volume +{{- if eq .Values.chain_type "public" }} - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}" imagePullPolicy: IfNotPresent name: snapshot-downloader @@ -181,6 +207,7 @@ spec: envFrom: - configMapRef: name: tezos-config +{{- end }} {{- if not .Values.is_invitation }} - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} imagePullPolicy: IfNotPresent @@ -188,31 +215,6 @@ spec: volumeMounts: - mountPath: /var/tezos name: var-volume -{{- end }} -{{- if eq .Values.chain_type "private" }} - - envFrom: - - configMapRef: - name: zerotier-config - image: "{{ .Values.tezos_k8s_images.zerotier }}" - imagePullPolicy: IfNotPresent - name: get-zerotier-ip - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /var/tezos - name: var-volume - - mountPath: /dev/net/tun - name: dev-net-tun - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name {{- end }} securityContext: fsGroup: 100 diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml index 55366b37f..3b247c99a 100644 --- a/charts/tezos/templates/node.yaml +++ b/charts/tezos/templates/node.yaml @@ -63,6 +63,31 @@ spec: name: var-volume {{- end }} initContainers: +{{- if eq .Values.chain_type "private" }} + - envFrom: + - configMapRef: + name: zerotier-config + image: "{{ .Values.tezos_k8s_images.zerotier }}" + imagePullPolicy: IfNotPresent + name: get-zerotier-ip + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /var/tezos + name: var-volume + - mountPath: /dev/net/tun + name: dev-net-tun + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{- end }} - image: {{ .Values.tezos_k8s_images.config_generator }} imagePullPolicy: IfNotPresent name: config-generator @@ -101,31 +126,6 @@ spec: volumeMounts: - mountPath: /var/tezos name: var-volume -{{- end }} -{{- if eq .Values.chain_type "private" }} - - envFrom: - - configMapRef: - name: zerotier-config - image: "{{ .Values.tezos_k8s_images.zerotier }}" - imagePullPolicy: IfNotPresent - name: get-zerotier-ip - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - privileged: true - volumeMounts: - - mountPath: /var/tezos - name: var-volume - - mountPath: /dev/net/tun - name: dev-net-tun - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name {{- end }} securityContext: fsGroup: 100 diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 9b8bdfc18..a3c8e2a5a 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -23,8 +23,8 @@ tezos_k8s_images: snapshot_url: https://mainnet.xtz-shots.io/rolling protocol: bootstrap_mutez: "4000000000000" - hash: PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo - command: 007-PsDELPH1 + hash: PtEdo2ZkT9oKpimTah6x2embF25oss54njMuPzkJTEi5RqfdZFA + command: 008-PtEdo2Zk genesis: block: "YOUR_GENESIS_CHAIN_ID_HERE" timestamp: '2021-02-12T17:02:24.061398+00:00' From 4a098f1928d5bb8b9ffe4c4305a24e41a2f1c441 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sat, 27 Feb 2021 21:49:02 -0800 Subject: [PATCH 11/17] don't put history mode in rolling if not public node later it will be parametrized --- config-generator/entrypoint.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index dc8d8effc..996643793 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -120,13 +120,11 @@ def get_node_config( "bootstrap-peers": bootstrap_peers, "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" ) }, - "shell": { - "history_mode": "rolling" - }, #"log": { "level": "debug"}, } if CHAIN_PARAMS["chain_type"] == "public": node_config["network"] = CHAIN_PARAMS["network"] + node_config["shell"] = { "history_mode": "rolling" } else: node_config["p2p"]["expected-proof-of-work"] = 0 node_config["network"] = { From 32b222ef95873dbb01328274dd09e275d36bcac5 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 11:21:00 -0800 Subject: [PATCH 12/17] point baker to correct data dir --- baker-endorser/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/baker-endorser/entrypoint.sh b/baker-endorser/entrypoint.sh index efcdccf76..0daf034e3 100755 --- a/baker-endorser/entrypoint.sh +++ b/baker-endorser/entrypoint.sh @@ -6,10 +6,11 @@ TEZ_VAR=/var/tezos TEZ_BIN=/usr/local/bin CLIENT_DIR="$TEZ_VAR/client" NODE_DIR="$TEZ_VAR/node" +NODE_DATA_DIR="$TEZ_VAR/node/data" proto_command=$(echo $CHAIN_PARAMS | jq -r '.proto_command') if [ "${DAEMON}" == "baker" ]; then - extra_args="with local node $NODE_DIR" + extra_args="with local node $NODE_DATA_DIR" fi POD_INDEX=$(echo $POD_NAME | sed -e s/tezos-baking-node-//) baker_account=$(echo $NODES | jq -r ".baking[${POD_INDEX}].bake_for") From bdd4f97e668159ec218c7fb2b93acd89df7e175f Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 13:17:10 -0800 Subject: [PATCH 13/17] make config container slimmer --- config-generator/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/config-generator/Dockerfile b/config-generator/Dockerfile index 9a2492678..8ff22b634 100644 --- a/config-generator/Dockerfile +++ b/config-generator/Dockerfile @@ -1,7 +1,12 @@ FROM tezos/tezos:master_328bc6c3_20210215194506 ENV PYTHONUNBUFFERED=1 -RUN sudo apk add --no-cache --virtual .build-deps jq gcc py3-pip python3-dev musl-dev libffi-dev make \ - && pip install base58 pynacl + +# Note: we install build deps for pip, then remove everything after pip install. +# Then we reinstall runtime deps at the end. +RUN sudo apk add --no-cache --virtual .build-deps gcc py3-pip python3-dev musl-dev libffi-dev make \ + && pip install base58 pynacl \ + && sudo apk del .build-deps gcc musl-dev libffi-dev make \ + && sudo apk add --no-cache --virtual install jq gcc python3 py3-pip COPY entrypoint.py / COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] From 14a92970b0a6979b5b7f61de84b536625b3c29af Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 14:24:25 -0800 Subject: [PATCH 14/17] revert custom version --- baker-endorser/Dockerfile | 2 +- chain-initiator/Dockerfile | 2 +- charts/tezos/values.yaml | 2 +- config-generator/Dockerfile | 2 +- snapshot-downloader/Dockerfile | 2 +- wait-for-bootstrap/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/baker-endorser/Dockerfile b/baker-endorser/Dockerfile index 0abb3ce22..38325f207 100644 --- a/baker-endorser/Dockerfile +++ b/baker-endorser/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master_328bc6c3_20210215194506 +FROM tezos/tezos:v8-release RUN sudo apk add jq bash COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/chain-initiator/Dockerfile b/chain-initiator/Dockerfile index 4c6e6ac4b..05c0cbd09 100644 --- a/chain-initiator/Dockerfile +++ b/chain-initiator/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master_328bc6c3_20210215194506 +FROM tezos/tezos:v8-release RUN sudo apk add jq COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index a3c8e2a5a..1acffb482 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -10,7 +10,7 @@ zerotier_config: # Images not part of the tezos-k8s repo go here images: - tezos: master_328bc6c3_20210215194506 + tezos: v8-release # Images that are part of the tezos-k8s repo go here tezos_k8s_images: baker_endorser: tezos-k8s-baker-endorser:dev diff --git a/config-generator/Dockerfile b/config-generator/Dockerfile index 8ff22b634..7fc9c8b96 100644 --- a/config-generator/Dockerfile +++ b/config-generator/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master_328bc6c3_20210215194506 +FROM tezos/tezos:v8-release ENV PYTHONUNBUFFERED=1 # Note: we install build deps for pip, then remove everything after pip install. diff --git a/snapshot-downloader/Dockerfile b/snapshot-downloader/Dockerfile index caba59ad4..71ca59bc6 100644 --- a/snapshot-downloader/Dockerfile +++ b/snapshot-downloader/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master_328bc6c3_20210215194506 +FROM tezos/tezos:v8-release RUN sudo apk add curl lz4 xz jq COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/wait-for-bootstrap/Dockerfile b/wait-for-bootstrap/Dockerfile index e4dc0ed67..7bf21b0cc 100644 --- a/wait-for-bootstrap/Dockerfile +++ b/wait-for-bootstrap/Dockerfile @@ -1,4 +1,4 @@ -FROM tezos/tezos:master_328bc6c3_20210215194506 +FROM tezos/tezos:v8-release RUN sudo apk add jq netcat-openbsd COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] From 7d851b2fe5c0a0c44e6bb6d0e45b00d3824a2459 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 14:26:46 -0800 Subject: [PATCH 15/17] lint --- config-generator/entrypoint.py | 101 ++++++++++++++++----------------- mkchain/tqchain/mkchain.py | 7 ++- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py index 996643793..42344cb2c 100644 --- a/config-generator/entrypoint.py +++ b/config-generator/entrypoint.py @@ -64,7 +64,7 @@ def main(): ) bootstrap_peers.extend(local_bootstrap_peers) if not bootstrap_peers: - bootstrap_peers=[f"tezos-baking-node-0.tezos-baking-node:9732"] + bootstrap_peers = [f"tezos-baking-node-0.tezos-baking-node:9732"] config_json = json.dumps( get_node_config( @@ -112,36 +112,37 @@ def get_node_config( net_addr=None, ): - node_config = { "data-dir": "/var/tezos/node/data", - "rpc": { - "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], - }, - "p2p": { - "bootstrap-peers": bootstrap_peers, - "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" ) - }, - #"log": { "level": "debug"}, - } + node_config = { + "data-dir": "/var/tezos/node/data", + "rpc": { + "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"], + }, + "p2p": { + "bootstrap-peers": bootstrap_peers, + "listen-addr": (net_addr + ":9732" if net_addr else "[::]:9732"), + }, + # "log": { "level": "debug"}, + } if CHAIN_PARAMS["chain_type"] == "public": node_config["network"] = CHAIN_PARAMS["network"] - node_config["shell"] = { "history_mode": "rolling" } + node_config["shell"] = {"history_mode": "rolling"} else: node_config["p2p"]["expected-proof-of-work"] = 0 node_config["network"] = { - "chain_name": chain_name, - "sandboxed_chain_name": "SANDBOXED_TEZOS", - "default_bootstrap_peers": [], - "genesis": { - "timestamp": timestamp, - "block": genesis_block, - "protocol": "PtYuensgYBb3G3x1hLLbCmcav8ue8Kyd2khADcL5LsT5R1hcXex" - }, - "genesis_parameters": { - "values": { - "genesis_pubkey": genesis_key, - }, + "chain_name": chain_name, + "sandboxed_chain_name": "SANDBOXED_TEZOS", + "default_bootstrap_peers": [], + "genesis": { + "timestamp": timestamp, + "block": genesis_block, + "protocol": "PtYuensgYBb3G3x1hLLbCmcav8ue8Kyd2khADcL5LsT5R1hcXex", + }, + "genesis_parameters": { + "values": { + "genesis_pubkey": genesis_key, }, - } + }, + } return node_config @@ -198,19 +199,22 @@ def generate_parameters_config(parameters_argv): namespace = parser.parse_args(parameters_argv) return vars(namespace) + # # If CHAIN_PARAMS["genesis_block"] hasn't been specified, we # generate a deterministic one. + def fill_in_missing_genesis_block(): print("Ensure that we have genesis_block") - if CHAIN_PARAMS["genesis_block"] == 'YOUR_GENESIS_CHAIN_ID_HERE': + if CHAIN_PARAMS["genesis_block"] == "YOUR_GENESIS_CHAIN_ID_HERE": print(" Generating missing genesis_block") seed = "foo" - gbk = blake2b(seed.encode(), digest_size=32).digest() + gbk = blake2b(seed.encode(), digest_size=32).digest() gbk_b58 = b58encode_check(b"\x01\x34" + gbk).decode("utf-8") CHAIN_PARAMS["genesis_block"] = gbk_b58 + # # flatten_accounts() turns ACCOUNTS into a more amenable data structure: # @@ -229,28 +233,30 @@ def fill_in_missing_genesis_block(): # If we then find that we have been asked to make more bakers # than accounts were specified, we create accounts of the form # -# baker +# baker # # and fill in the details appropriately. + def flatten_accounts(): accounts = {} - for name, type, key in [[a["name"],a["type"],a["key"]] for a in ACCOUNTS]: + for name, type, key in [[a["name"], a["type"], a["key"]] for a in ACCOUNTS]: if name in accounts: if type in accounts[name]: print(" WARNING: key specified twice! " + name + ":" + type) else: accounts[name][type] = key else: - accounts[name] = { type : key } - i=0 + accounts[name] = {type: key} + i = 0 for i, node in enumerate(CHAIN_PARAMS["nodes"]["baking"]): acct = node.get("bake_for", "baker" + str(i)) if acct not in accounts: - print(" Creating specified but missing account " + acct); + print(" Creating specified but missing account " + acct) accounts[acct] = {} return accounts + # # import_keys() creates three files in /var/tezos/client which specify # the keys for each of the accounts: secret_keys, public_keys, and @@ -270,9 +276,10 @@ def flatten_accounts(): # are stable because we take care not to use any information that is not # specified in the _values.yaml file in the seed used to generate them. -edsk = b"\x0d\x0f\x3a\x07"; +edsk = b"\x0d\x0f\x3a\x07" edpk = b"\x0d\x0f\x25\xd9" -tz1 = b"\x06\xa1\x9f" +tz1 = b"\x06\xa1\x9f" + def import_keys(): print("Importing keys") @@ -312,39 +319,29 @@ def import_keys(): pkh = blake2b(pk, digest_size=20).digest() - pk_b58 = b58encode_check(edpk + pk).decode("utf-8") + pk_b58 = b58encode_check(edpk + pk).decode("utf-8") pkh_b58 = b58encode_check(tz1 + pkh).decode("utf-8") if sk != None: print(" Appending secret key") sk_b58 = b58encode_check(edsk + sk).decode("utf-8") - secret_keys.append({ - "name" : name, - "value": "unencrypted:" + sk_b58 - }) + secret_keys.append({"name": name, "value": "unencrypted:" + sk_b58}) print(" Appending public key") - public_keys.append({ - "name" : name, - "value": { - "locator" : "unencrypted:" + pk_b58, - "key": pk_b58 - } - }) + public_keys.append( + {"name": name, "value": {"locator": "unencrypted:" + pk_b58, "key": pk_b58}} + ) print(" Appending public key hash") - public_key_hashs.append({ - "name" : name, - "value": pkh_b58 - }) + public_key_hashs.append({"name": name, "value": pkh_b58}) print(" Writing " + tezdir + "/secret_keys") json.dump(secret_keys, open(tezdir + "/secret_keys", "w"), indent=4) print(" Writing " + tezdir + "/public_keys") json.dump(public_keys, open(tezdir + "/public_keys", "w"), indent=4) print(" Writing " + tezdir + "/public_key_hashs") - json.dump(public_key_hashs, open(tezdir + "/public_key_hashs", "w"), - indent=4) + json.dump(public_key_hashs, open(tezdir + "/public_key_hashs", "w"), indent=4) + if __name__ == "__main__": main() diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index afee511ad..07faad45c 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -71,9 +71,10 @@ def get_genesis_vanity_chain_id(docker_image, seed_len=16): "default": 1, "type": int, }, - "chain_type": {"help": "Type of chain: public, private, isolated", + "chain_type": { + "help": "Type of chain: public, private, isolated", "default": "isolated", - }, + }, "network": {"help": "Name of the tezos public network to connect to "}, "zerotier_network": {"help": "Zerotier network id for external chain access"}, "zerotier_token": {"help": "Zerotier token for external chain access"}, @@ -195,7 +196,7 @@ def main(): } if base_constants["chain_type"] == "public": - base_constants["network"] = ( args.network if args.network else "mainnet" ) + base_constants["network"] = args.network if args.network else "mainnet" accounts = {"secret": [], "public": []} if old_create_values.get("accounts", None): From 0ab21fcca643c6f629c540d65d0f365ff24910eb Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 14:52:52 -0800 Subject: [PATCH 16/17] restore default values.yaml to isolated chain --- charts/tezos/values.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 1acffb482..03904d7b2 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -2,15 +2,14 @@ baker: true chain_name: YOUR_CHAIN_NAME_HERE is_invitation: false rpc_auth: false -chain_type: public -network: edo2net +chain_type: isolated zerotier_config: zerotier_network: YOUR_NETWORK_ID_HERE zerotier_token: YOUR_ZEROTIER_TOKEN_HERE # Images not part of the tezos-k8s repo go here images: - tezos: v8-release + tezos: tezos/tezos:v8-release # Images that are part of the tezos-k8s repo go here tezos_k8s_images: baker_endorser: tezos-k8s-baker-endorser:dev From 9fcb451b9143c63e15a138d471662e0186a867b8 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 1 Mar 2021 18:00:54 -0800 Subject: [PATCH 17/17] move wait-for-bootstrap back to the first init container do not download snapshot for nodes in isolated chains --- charts/tezos/templates/baker.yaml | 16 ++++++++-------- charts/tezos/templates/node.yaml | 16 +++++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml index d150bc686..81e36e679 100644 --- a/charts/tezos/templates/baker.yaml +++ b/charts/tezos/templates/baker.yaml @@ -150,6 +150,14 @@ spec: name: var-volume {{- end }} initContainers: +{{- if not .Values.is_invitation }} + - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} + imagePullPolicy: IfNotPresent + name: wait-for-bootstrap + volumeMounts: + - mountPath: /var/tezos + name: var-volume +{{- end }} {{- if eq .Values.chain_type "private" }} - envFrom: - configMapRef: @@ -207,14 +215,6 @@ spec: envFrom: - configMapRef: name: tezos-config -{{- end }} -{{- if not .Values.is_invitation }} - - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} - imagePullPolicy: IfNotPresent - name: wait-for-bootstrap - volumeMounts: - - mountPath: /var/tezos - name: var-volume {{- end }} securityContext: fsGroup: 100 diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml index 3b247c99a..cb5c3de39 100644 --- a/charts/tezos/templates/node.yaml +++ b/charts/tezos/templates/node.yaml @@ -63,6 +63,14 @@ spec: name: var-volume {{- end }} initContainers: +{{- if not .Values.is_invitation }} + - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} + imagePullPolicy: IfNotPresent + name: wait-for-bootstrap + volumeMounts: + - mountPath: /var/tezos + name: var-volume +{{- end }} {{- if eq .Values.chain_type "private" }} - envFrom: - configMapRef: @@ -108,6 +116,7 @@ spec: name: config-volume - mountPath: /var/tezos name: var-volume +{{- if eq .Values.chain_type "public" }} - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}" imagePullPolicy: IfNotPresent name: snapshot-downloader @@ -119,13 +128,6 @@ spec: envFrom: - configMapRef: name: tezos-config -{{- if not .Values.is_invitation }} - - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }} - imagePullPolicy: IfNotPresent - name: wait-for-bootstrap - volumeMounts: - - mountPath: /var/tezos - name: var-volume {{- end }} securityContext: fsGroup: 100