diff --git a/pulumi/Pulumi.yaml b/pulumi/Pulumi.yaml
new file mode 100644
index 0000000..d2beeb7
--- /dev/null
+++ b/pulumi/Pulumi.yaml
@@ -0,0 +1,2 @@
+name: qsfs
+runtime: python
diff --git a/pulumi/README.md b/pulumi/README.md
new file mode 100644
index 0000000..eb28f9c
--- /dev/null
+++ b/pulumi/README.md
@@ -0,0 +1,89 @@
+# Deploy QSFS with Pulumi
+
+This is a Pulumi deployment script in Python that fully automates the setup of a QSFS instance. The following steps are required to use this script:
+
+1. Install Pulumi and Python on your system
+2. Use Pip to install the Python dependencies
+3. Copy and edit vars.py and zstor_config.base.toml
+
+Only Linux and MacOS are supported. If you run Windows, I'd recommend equipping yourself with a WSL environment.
+
+## Install Pulumi and Python
+
+We won't cover the details here. Probably your system already has `python3`.
+
+For Pulumi, check here: https://www.pulumi.com/docs/iac/download-install/
+
+## Install Python dependencies
+
+We need some Python packages to make this work. Using a venv is recommended.
+
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install pulumi pulumi_random pulumi_command pulumi_threefold
+```
+
+## Prep config
+
+Two config files are needed. Examples are included here. Copy the examples to the expected paths, then edit the files according to your needs.
+
+```
+cp vars.example.py vars.py
+cp zstor_config.base.example.toml zstor_config.base.toml
+
+$EDITOR vars.py
+$EDITOR zstor_config.base.toml
+```
+
+## Deploy
+
+Prior to using Pulumi, you need to login. There are some options here, which you can read about, but the simplest thing is to just use `--local`:
+
+```
+pulumi login --local
+```
+
+Now we can bring up the deployment. Create a stack when prompted with your name of choice.
+
+```
+pulumi up
+```
+
+If you want to destroy the deployment, bring it down like this:
+
+```
+pulumi down
+```
+
+## Replacing backends
+
+If you want to replace any data or metadata backends, just edit `vars.py` and run `pulumi up` again. Note that this is a destructive operation and any backends not present in the new config will be decomissioned. Data loss is possible if too many backends are decommissioned at one time without rebuilding the data. You must have the minimal shard count available to be able to reconstruct the data.
+
+After running `pulumi up` with the new config, the Pulumi script will automatically upload an updated Zstor config file to the VM. However, Zstor will not start using the new config automatically. You either need to restart Zstor or perform a hot reload of the config by sending the SIGUSR1 signal to Zstor:
+
+```
+pkill zstor -SIGUSR1
+```
+
+Once the new config is loaded, Zstor will automatically start writing data or metadata to the new backends to restore the desired shard count for each stored file. This can take up to ten minutes to be triggered.
+
+You can check the progress of rebuilding using the Zstor `status` command:
+
+```
+zstor -c /etc/zstor-default.toml status
+```
+
+## Recover to new VM
+
+If you need to replace the frontend VM for any reason, such as a node outage, follow these steps. Any data that has been uploaded to the backends can be recovered into the new VM. Any data that was not yet uploaded to the backends will be lost.
+
+1. Update the `vars.py` file and set `VM_NODE` to the new node id
+2. Destroy the old VM and deploy the new VM by running `pulumi up`
+3. SSH to the new VM and run the recovery script:
+
+```
+bash /root/scripts/recover.sh
+```
+
+If all went well, your files should appear under the mount point, `/mnt/qsfs`.
diff --git a/pulumi/__main__.py b/pulumi/__main__.py
new file mode 100644
index 0000000..c520b49
--- /dev/null
+++ b/pulumi/__main__.py
@@ -0,0 +1,324 @@
+import os
+import shutil
+import textwrap
+from pathlib import Path
+
+import pulumi
+import pulumi_random
+import pulumi_command
+import pulumi_threefold as threefold
+
+# It's up to the user to create their own vars.py before trying to deploy
+try:
+    from vars import (
+        MNEMONIC,
+        NETWORK,
+        SSH_KEY_PATH,
+        VM_NODE,
+        META_NODES,
+        DATA_NODES,
+        DATA_SIZE,
+        ZDB_CONNECTION,
+        SSH_CONNECTION,
+    )
+except ModuleNotFoundError:
+    exit("vars.py not found. Exiting.")
+
+# Same for the base zstor config. Exit if the user didn't provide this
+ZSTOR_CONFIG_BASE = "zstor_config.base.toml"
+ZSTOR_CONFIG_PATH = "zstor_config.toml"
+# This path is hard coded in the Zdb hook script
+ZSTOR_CONFIG_REMOTE = "/etc/zstor-default.toml"
+
+if not os.path.exists(ZSTOR_CONFIG_BASE):
+    exit("zstor_config.base.toml not found. Exiting.")
+
+
+# Path of the script that will run on the deployed VM after deployment
+# Installs needed binaries and starts up all the services
+POST_DEPLOY_SCRIPT = "post_deploy.sh"
+
+# If a node has IPv6, then it will be the first IP in the zdb IP list
+# Mycelium will always be last, but this could be index 1 or 2
+ZDB_IP6_INDEX = 0
+ZDB_MYC_INDEX = -1
+
+# From here are all the parameters for the deployment
+MNEMONIC = MNEMONIC if MNEMONIC else os.environ.get("MNEMONIC")
+NETWORK = NETWORK if NETWORK else os.environ.get("NETWORK")
+
+ssh_key_path = os.path.expanduser(SSH_KEY_PATH)
+
+with open(ssh_key_path, "r") as file:
+    ssh_private_key = file.read()
+
+with open(ssh_key_path + ".pub", "r") as file:
+    ssh_public_key = file.read()
+
+FLIST = "https://hub.grid.tf/tf-official-apps/threefoldtech-ubuntu-22.04.flist"
+CPU = 1
+RAM = 2048  # MB
+ROOTFS = 1024 * 15  # MB
+NET_NAME = "net"
+
+META_SIZE = 1
+
+# Generate separate secrets for Zstor key and Zdb namespaces passwords
+zstor_key = pulumi_random.RandomBytes("zstor_key", length=32)
+zdb_pw = pulumi_random.RandomPassword("zdb_pw", length=20)
+
+if ZDB_CONNECTION == "ipv6":
+    ZDB_IP_INDEX = ZDB_IP6_INDEX
+elif ZDB_CONNECTION == "mycelium":
+    ZDB_IP_INDEX = ZDB_MYC_INDEX
+
+# CopyToRemote requires that the path used contains some file from the start, so
+# we just put an empty one there if needed
+Path(ZSTOR_CONFIG_PATH).touch()
+
+provider = threefold.Provider(
+    "provider",
+    mnemonic=MNEMONIC,
+    network=NETWORK,
+)
+
+# Deploying a VM is optional. Some users might want to use an existing VM or
+# another system for their QFS frontend
+if VM_NODE is not None:
+    network = threefold.Network(
+        "network",
+        name=NET_NAME,
+        description="A network",
+        nodes=[VM_NODE],
+        ip_range="10.1.0.0/16",
+        # With mycelium enabled, we can't redeploy the vm
+        # https://github.com/threefoldtech/pulumi-threefold/issues/552
+        # Maybe it's okay though if we use separate deployements for vm and zdbs?
+        # mycelium=True,
+        opts=pulumi.ResourceOptions(provider=provider),
+    )
+
+    vm_deployment = threefold.Deployment(
+        "vm_deployment",
+        node_id=VM_NODE,
+        name="vm",
+        network_name=NET_NAME,
+        vms=[
+            threefold.VMInputArgs(
+                name="vm",
+                node_id=VM_NODE,
+                flist=FLIST,
+                entrypoint="/sbin/zinit init",
+                network_name=NET_NAME,
+                cpu=CPU,
+                memory=RAM,
+                rootfs_size=ROOTFS,
+                # mycelium=True,
+                planetary=True,
+                public_ip6=True,
+                env_vars={
+                    "SSH_KEY": ssh_public_key,
+                },
+            )
+        ],
+        opts=pulumi.ResourceOptions(provider=provider, depends_on=[network]),
+    )
+else:
+    vm_deployment = None
+
+zdb_nodes = set(META_NODES + DATA_NODES)
+zdb_deployments = []
+
+for node in zdb_nodes:
+    zdbs = []
+    if node in DATA_NODES:
+        zdbs.append(
+            threefold.ZDBInputArgs(
+                name="data" + str(node),
+                size=DATA_SIZE,
+                mode="seq",
+                password=zdb_pw.result,
+            )
+        )
+    if node in META_NODES:
+        zdbs.append(
+            threefold.ZDBInputArgs(
+                name="meta" + str(node),
+                size=META_SIZE,
+                mode="user",
+                password=zdb_pw.result,
+            )
+        )
+
+    zdb_deployments.append(
+        threefold.Deployment(
+            "zdb_deployment" + str(node),
+            node_id=node,
+            name="node" + str(node),
+            zdbs=zdbs,
+            opts=pulumi.ResourceOptions(provider=provider),
+        )
+    )
+
+
+def make_ssh_connection(vm):
+    if SSH_CONNECTION == "mycelium":
+        ssh_ip = vm["mycelium_ip"]
+    elif SSH_CONNECTION == "ipv6":
+        ssh_ip = vm["computed_ip6"].apply(lambda ip6: ip6.split("/")[0])
+
+    return pulumi_command.remote.ConnectionArgs(
+        host=ssh_ip,
+        user="root",
+        private_key=ssh_private_key,
+    )
+
+
+def make_zstor_config(args):
+    # Changes to the zdb backends mean that we need to regenerate the config
+    # file. Here we always choose a new local path and leave the old files
+    # around just in case
+    i = 1
+    while os.path.exists(path := ZSTOR_CONFIG_PATH + "." + str(i)):
+        i += 1
+
+    shutil.copy(ZSTOR_CONFIG_BASE, path)
+
+    vm = args["vm"]
+    meta_zdbs = []
+    data_zdbs = []
+    for zdbs in args["zdbs"]:
+        for zdb in zdbs:
+            if "meta" in zdb["namespace"]:
+                meta_zdbs.append(zdb)
+            else:
+                data_zdbs.append(zdb)
+    meta_zdbs = sorted(meta_zdbs, key=lambda z: z["namespace"].split("-")[-1])
+    data_zdbs = sorted(data_zdbs, key=lambda z: z["namespace"].split("-")[-1])
+
+    with open(path, "a") as file:
+        encryption_config = f"""
+        [encryption]
+        algorithm = "AES"
+        key = "{args['zstor_key']}"
+
+        [meta.config.encryption]
+        algorithm = "AES"
+        key = "{args['zstor_key']}"
+        """
+        file.write(textwrap.dedent(encryption_config))
+        for zdb in meta_zdbs:
+            ip = zdb["ips"][ZDB_IP_INDEX]
+            ns = zdb["namespace"]
+            file.write("[[meta.config.backends]]\n")
+            file.write(f'address = "[{ip}]:9900"\n')
+            file.write(f'namespace = "{ns}"\n')
+            file.write(f'password = "{args['zdb_pw']}"\n\n')
+
+        file.write("[[groups]]\n")
+        for zdb in data_zdbs:
+            ip = zdb["ips"][ZDB_IP_INDEX]
+            ns = zdb["namespace"]
+            file.write("[[groups.backends]]\n")
+            file.write(f'address = "[{ip}]:9900"\n')
+            file.write(f'namespace = "{ns}"\n')
+            file.write(f'password = "{args['zdb_pw']}"\n\n')
+
+    # This way the current file is always in the same place and we get around
+    # the fact that it's not possible to return a path from this function and
+    # use it as a FileAsset, because you can't pass an Output to FileAsset
+    if not open(path).read() == open(ZSTOR_CONFIG_PATH).read():
+        shutil.copy(path, ZSTOR_CONFIG_PATH)
+    else:
+        # We end up regenerating the same file from time to time for reasons
+        # that may or may not be unavoidable. For now, just delete duplicates
+        os.remove(path)
+
+
+zstor_config_output = pulumi.Output.all(
+    vm=vm_deployment.vms_computed[0],
+    zdbs=[d.zdbs_computed for d in zdb_deployments],
+    zstor_key=zstor_key.hex,
+    zdb_pw=zdb_pw.result,
+).apply(make_zstor_config)
+
+if vm_deployment:
+    vm = vm_deployment.vms_computed[0]
+    conn = make_ssh_connection(vm)
+    depends = []
+
+    copy_zstor_config = pulumi_command.remote.CopyToRemote(
+        "copy_zstor_config",
+        connection=conn,
+        source=pulumi.FileAsset(ZSTOR_CONFIG_PATH),
+        remote_path=ZSTOR_CONFIG_REMOTE,
+        # Without this trigger, a new upload isn't triggered when the VM is
+        # replaced. However, the file on an existing VM gets updated just with
+        # zstor_config_output in the depends_on list
+        triggers=[conn.host],
+        opts=pulumi.ResourceOptions(depends_on=[zstor_config_output, vm_deployment]),
+    )
+
+    if os.path.isfile("prometheus.yaml"):
+        depends.append(
+            pulumi_command.remote.CopyToRemote(
+                "copy_prometheus",
+                connection=conn,
+                source=pulumi.FileAsset("prometheus.yaml"),
+                remote_path="/etc/prometheus.yaml",
+                triggers=[conn.host],
+            )
+        )
+
+    # In case we want to test our own zstor binary, such as a prebuild
+    if os.path.isfile("zstor"):
+        depends.append(
+            pulumi_command.remote.CopyToRemote(
+                "copy_zstor_binary",
+                connection=conn,
+                source=pulumi.FileAsset("zstor"),
+                remote_path="/usr/bin/zstor",
+                triggers=[conn.host],
+            )
+        )
+
+    # We put the zinit files under /root to start, so that the services don't get
+    # started accidentally on reboot. In the case of recovering on a new VM, we
+    # need to ensure some other steps are completed first
+    copy_zinit = pulumi_command.remote.CopyToRemote(
+        "copy_zinit",
+        connection=conn,
+        source=pulumi.FileArchive("zinit/"),
+        remote_path="/root/zinit/",
+        triggers=[conn.host],
+    )
+
+    copy_scripts = pulumi_command.remote.CopyToRemote(
+        "copy_scripts",
+        connection=conn,
+        source=pulumi.FileArchive("scripts/"),
+        remote_path="/root/scripts/",
+        triggers=[conn.host],
+    )
+
+    depends.append(copy_scripts)
+
+    prep_vm = pulumi_command.remote.Command(
+        "prep_vm",
+        connection=conn,
+        create="bash /root/scripts/prep_vm.sh 2>&1 | tee > /var/log/prep_vm.log",
+        triggers=[conn.host],
+        opts=pulumi.ResourceOptions(depends_on=depends),
+    )
+
+    depends.extend([prep_vm, copy_zinit, copy_zstor_config])
+    pulumi_command.remote.Command(
+        "activate_qsfs",
+        connection=conn,
+        create="bash /root/scripts/activate_qsfs.sh 2>&1 | tee > /var/log/activate_qsfs.log",
+        update="",
+        opts=pulumi.ResourceOptions(depends_on=depends),
+    )
+
+pulumi.export("mycelium_ip", vm.mycelium_ip)
+pulumi.export("pub_ipv6", vm.computed_ip6)
diff --git a/pulumi/prometheus.example.yaml b/pulumi/prometheus.example.yaml
new file mode 100644
index 0000000..298eb12
--- /dev/null
+++ b/pulumi/prometheus.example.yaml
@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 60s
+  external_labels:
+    origin_prometheus: prometheus01
+remote_write:
+  - url: https://your-prometheus.url
+    basic_auth:
+      username: user
+      password: password
+scrape_configs:
+  - job_name: zstor
+    static_configs:
+      - targets: ["localhost:9200"]
+  - job_name: node-exporter-zstor
+    static_configs:
+      - targets: ["localhost:9100"]
+  - job_name: pushgateway-zstor
+    static_configs:
+      - targets: ["localhost:9091"]
+    honor_labels: true
diff --git a/pulumi/scripts/activate_qsfs.sh b/pulumi/scripts/activate_qsfs.sh
new file mode 100755
index 0000000..c7591e7
--- /dev/null
+++ b/pulumi/scripts/activate_qsfs.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# This script starts up the qsfs, ensuring the mount point exists
+
+set -x
+
+# Primitive idempotency
+zinit | grep -q zstor && exit
+
+echo
+echo Creating Zdbfs mountpoint
+mkdir -p /mnt/qsfs
+
+echo
+echo Copying zinit service files
+cp /root/zinit/zstor.yaml /etc/zinit
+cp /root/zinit/zdb.yaml /etc/zinit
+cp /root/zinit/zdbfs.yaml /etc/zinit
+cp /root/zinit/retry-uploads.yaml /etc/zinit
+
+echo
+echo Starting up zinit services
+zinit monitor zstor
+zinit monitor zdb
+zinit monitor zdbfs
+zinit monitor retry-uploads
+
+if [ -f /etc/prometheus.yaml ]; then
+    echo
+    echo Copying Prometheus zinit service files
+    cp /root/zinit/node-exporter.yaml /etc/zinit
+    cp /root/zinit/prometheus.yaml /etc/zinit
+
+    echo
+    echo Starting up Prometheus zinit services
+    zinit monitor node-exporter
+    zinit monitor prometheus
+fi
diff --git a/pulumi/scripts/prep_vm.sh b/pulumi/scripts/prep_vm.sh
new file mode 100755
index 0000000..83ae137
--- /dev/null
+++ b/pulumi/scripts/prep_vm.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This script installs all binaries and scripts needed for QSFS. It doesn't actually start up the services though
+
+set -x
+
+# Grab binaries and hook script. Make sure that all are executable
+# We check first if the files exist, to support testing other builds by
+# uploading them into the VM before running this script
+if ! [ -f /usr/local/bin/zdbfs ]; then
+    wget -O /usr/local/bin/zdbfs https://github.com/threefoldtech/0-db-fs/releases/download/v0.1.11/zdbfs-0.1.11-amd64-linux-static
+fi
+
+if ! [ -f /usr/local/bin/zdb ]; then
+    wget -O /usr/local/bin/zdb https://github.com/threefoldtech/0-db/releases/download/v2.0.8/zdb-2.0.8-linux-amd64-static
+fi
+
+if ! [ -f /usr/local/bin/zdb-hook.sh ]; then
+    wget -O /usr/local/bin/zdb-hook.sh https://raw.githubusercontent.com/threefoldtech/quantum-storage/master/lib/zdb-hook.sh
+fi
+
+if ! [ -f /usr/local/bin/retry-uploads.sh ]; then
+    wget -O /usr/local/bin/retry-uploads.sh https://raw.githubusercontent.com/threefoldtech/quantum-storage/master/lib/retry-uploads.sh
+fi
+
+if ! [ -f /bin/zstor ]; then
+    wget -O /bin/zstor https://github.com/threefoldtech/0-stor_v2/releases/download/v0.4.0/zstor_v2-x86_64-linux-musl
+fi
+
+echo
+echo Setting permissions for downloaded binaries
+chmod +x /usr/local/bin/* /bin/zstor
+
+if [ -f /etc/prometheus.yaml ]; then
+    echo
+    echo Installing Prometheus
+    apt update
+    apt install -y prometheus prometheus-pushgateway curl
+fi
diff --git a/pulumi/scripts/recover.sh b/pulumi/scripts/recover.sh
new file mode 100755
index 0000000..3824a56
--- /dev/null
+++ b/pulumi/scripts/recover.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# This script is for recovering an existing QSFS onto a new VM
+
+echo
+echo "Creating QSFS mount point at /mnt/qsfs..."
+mkdir -p /mnt/qsfs
+
+echo
+echo "Starting zstor and zdb services..."
+cp /root/zinit/* /etc/zinit
+zinit monitor zstor
+zinit monitor zdb
+
+# The temp namespace is not backed up, so we just create it manually
+echo
+echo "Installing redis-cli..."
+apt update && apt install -y redis
+
+echo
+echo "Setting up temp namespace..."
+redis-cli -p 9900 NSNEW zdbfs-temp
+redis-cli -p 9900 NSSET zdbfs-temp password hello
+redis-cli -p 9900 NSSET zdbfs-temp public 0
+redis-cli -p 9900 NSSET zdbfs-temp mode seq
+
+echo
+echo "Recovering metadata indexes..."
+zstor -c /etc/zstor-default.toml retrieve --file /data/index/zdbfs-meta/zdb-namespace
+i=0
+while true; do
+  result=$(zstor -c /etc/zstor-default.toml retrieve --file /data/index/zdbfs-meta/i$i 2>&1)
+  if echo $result | grep -q error
+    then break
+  fi
+  i=$((i+1))
+done
+
+echo
+echo "Retrieving latest metadata data file..."
+last_meta_index=$(ls /data/index/zdbfs-meta | tr -d i | sort -n | tail -n 1)
+zstor -c /etc/zstor-default.toml retrieve --file /data/data/zdbfs-meta/d$last_meta_index
+
+echo
+echo "Recovering data indexes..."
+zstor -c /etc/zstor-default.toml retrieve --file /data/index/zdbfs-data/zdb-namespace
+i=0
+while true; do
+  result=$(zstor -c /etc/zstor-default.toml retrieve --file /data/index/zdbfs-data/i$i 2>&1)
+  if echo $result | grep -q error
+    then break
+  fi
+  i=$((i+1))
+done
+
+echo
+echo "Retrieving latest data data file..."
+last_data_index=$(ls /data/index/zdbfs-data | tr -d i | sort -n | tail -n 1)
+zstor -c /etc/zstor-default.toml retrieve --file /data/data/zdbfs-data/d$last_data_index
+
+
+# Start zdbfs
+echo
+echo "Starting ZDBFS service..."
+zinit monitor zdbfs
diff --git a/pulumi/tests/rebuild/Pulumi.yaml b/pulumi/tests/rebuild/Pulumi.yaml
new file mode 100644
index 0000000..096294d
--- /dev/null
+++ b/pulumi/tests/rebuild/Pulumi.yaml
@@ -0,0 +1,2 @@
+name: qsfs-rebuild-test
+runtime: python
diff --git a/pulumi/tests/rebuild/README.md b/pulumi/tests/rebuild/README.md
new file mode 100644
index 0000000..f810a4b
--- /dev/null
+++ b/pulumi/tests/rebuild/README.md
@@ -0,0 +1,19 @@
+This is meant to be a fully automated test of the rebuild/repair system in Zstor.
+
+It does these steps:
+
+1. Deploy a QSFS using an original configuration
+2. Write some data into the QSFS (random files)
+3. Replace one of the original Zdbs with a new one on a different node (do this for both data and metadata)
+4. Upload a new config file to the frontend VM and try to hot reload the config using SIGUSR1
+5. Check the `status` output from zstor to see if some data has been written to the new backends
+
+Perhaps a better test would be to force zstor to rebuild the data from the new backend, by blocking access to enough of the original backends that using the new backend is necessary to fulfill the required shard count to rebuild.
+
+To use it, just:
+
+```
+./run.sh
+```
+
+This runs a set of scripts in the correct order. You can also run the scripts individually and inspect the state step by step.
diff --git a/pulumi/tests/rebuild/__main__.py b/pulumi/tests/rebuild/__main__.py
new file mode 120000
index 0000000..74b2cfc
--- /dev/null
+++ b/pulumi/tests/rebuild/__main__.py
@@ -0,0 +1 @@
+../../__main__.py
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/cleanup.sh b/pulumi/tests/rebuild/cleanup.sh
new file mode 100755
index 0000000..57e1a25
--- /dev/null
+++ b/pulumi/tests/rebuild/cleanup.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rm zstor_config.toml*
+rm md5s_original
+rm vars.py
diff --git a/pulumi/tests/rebuild/run.sh b/pulumi/tests/rebuild/run.sh
new file mode 100755
index 0000000..6b30190
--- /dev/null
+++ b/pulumi/tests/rebuild/run.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+source ../../venv/bin/activate
+
+../test-scripts-local/deploy.sh
+../test-scripts-local/upload_remote_scripts.sh
+../test-scripts-local/create_data.sh
+../test-scripts-local/redeploy.sh
+../test-scripts-local/recover.sh
+../test-scripts-local/upload_remote_scripts.sh
+../test-scripts-local/remove_and_rebuild.sh
+../test-scripts-local/destroy.sh
+./cleanup.sh
diff --git a/pulumi/tests/rebuild/scripts b/pulumi/tests/rebuild/scripts
new file mode 120000
index 0000000..11aee1d
--- /dev/null
+++ b/pulumi/tests/rebuild/scripts
@@ -0,0 +1 @@
+../../scripts/
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/vars.new.py b/pulumi/tests/rebuild/vars.new.py
new file mode 100644
index 0000000..96afbfb
--- /dev/null
+++ b/pulumi/tests/rebuild/vars.new.py
@@ -0,0 +1,24 @@
+# These are the new values used in the test
+
+MNEMONIC = ""
+NETWORK = "main"
+
+SSH_KEY_PATH = "~/.ssh/id_rsa"
+
+# Node to deploy VM on. Can overlap with Zdb nodes or not, doesn't matter
+VM_NODE = 1
+
+# Nodes to deploy Zdbs on
+META_NODES = [8, 10, 11, 24]
+DATA_NODES = META_NODES
+
+# Size of each data backend Zdb
+DATA_SIZE = 1
+
+# Network used to connect to the backend zdbs
+# ZDB_CONNECTION = "mycelium"
+ZDB_CONNECTION = "ipv6"
+
+# Network used for SSH connection
+# SSH_CONNECTION = "mycelium"
+SSH_CONNECTION = "ipv6"
diff --git a/pulumi/tests/rebuild/vars.original.py b/pulumi/tests/rebuild/vars.original.py
new file mode 100644
index 0000000..e4d027d
--- /dev/null
+++ b/pulumi/tests/rebuild/vars.original.py
@@ -0,0 +1,24 @@
+# These are the original values used in the test
+
+MNEMONIC = ""
+NETWORK = "main"
+
+SSH_KEY_PATH = "~/.ssh/id_rsa"
+
+# Node to deploy VM on. Can overlap with Zdb nodes or not, doesn't matter
+VM_NODE = 13
+
+# Nodes to deploy Zdbs on
+META_NODES = [8, 10, 11, 13]
+DATA_NODES = META_NODES
+
+# Size of each data backend Zdb
+DATA_SIZE = 1
+
+# Network used to connect to the backend zdbs
+# ZDB_CONNECTION = "mycelium"
+ZDB_CONNECTION = "ipv6"
+
+# Network used for SSH connection
+# SSH_CONNECTION = "mycelium"
+SSH_CONNECTION = "ipv6"
diff --git a/pulumi/tests/rebuild/zinit/node-exporter.yaml b/pulumi/tests/rebuild/zinit/node-exporter.yaml
new file mode 120000
index 0000000..d72c19d
--- /dev/null
+++ b/pulumi/tests/rebuild/zinit/node-exporter.yaml
@@ -0,0 +1 @@
+../../../zinit/node-exporter.yaml
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/zinit/prometheus.yaml b/pulumi/tests/rebuild/zinit/prometheus.yaml
new file mode 120000
index 0000000..f5ffa37
--- /dev/null
+++ b/pulumi/tests/rebuild/zinit/prometheus.yaml
@@ -0,0 +1 @@
+../../../zinit/prometheus.yaml
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/zinit/zdb.yaml b/pulumi/tests/rebuild/zinit/zdb.yaml
new file mode 100644
index 0000000..9e56d12
--- /dev/null
+++ b/pulumi/tests/rebuild/zinit/zdb.yaml
@@ -0,0 +1,10 @@
+exec: |
+  /usr/local/bin/zdb \
+    --index /data/index \
+    --data /data/data \
+    --logfile /var/log/zdb.log \
+    --datasize 67108864 \
+    --hook /usr/local/bin/zdb-hook.sh \
+    --rotate 10
+shutdown_timeout: 60
+after: [zstor]
diff --git a/pulumi/tests/rebuild/zinit/zdbfs.yaml b/pulumi/tests/rebuild/zinit/zdbfs.yaml
new file mode 120000
index 0000000..c26e599
--- /dev/null
+++ b/pulumi/tests/rebuild/zinit/zdbfs.yaml
@@ -0,0 +1 @@
+../../../zinit/zdbfs.yaml
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/zinit/zstor.yaml b/pulumi/tests/rebuild/zinit/zstor.yaml
new file mode 120000
index 0000000..b37e568
--- /dev/null
+++ b/pulumi/tests/rebuild/zinit/zstor.yaml
@@ -0,0 +1 @@
+../../../zinit/zstor.yaml
\ No newline at end of file
diff --git a/pulumi/tests/rebuild/zstor_config.base.toml b/pulumi/tests/rebuild/zstor_config.base.toml
new file mode 100644
index 0000000..76dad34
--- /dev/null
+++ b/pulumi/tests/rebuild/zstor_config.base.toml
@@ -0,0 +1,20 @@
+minimal_shards = 2
+expected_shards = 4
+redundant_groups = 0
+redundant_nodes = 0
+root = "/"
+zdbfs_mountpoint = "/mnt/qsfs"
+socket = "/tmp/zstor.sock"
+prometheus_port = 9100
+zdb_data_dir_path = "/data/data/zdbfs-data/"
+max_zdb_data_dir_size = 2560
+
+[compression]
+algorithm = "snappy"
+
+[meta]
+type = "zdb"
+
+[meta.config]
+prefix = "zstor-meta"
+
diff --git a/pulumi/tests/test-scripts-local/README.md b/pulumi/tests/test-scripts-local/README.md
new file mode 100644
index 0000000..e305e1e
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/README.md
@@ -0,0 +1 @@
+This folder contains scripts that run locally on the machine orchestrating the test. They are responsible for creating, updating, and destroying the deployment.
diff --git a/pulumi/tests/test-scripts-local/create_data.sh b/pulumi/tests/test-scripts-local/create_data.sh
new file mode 100755
index 0000000..e127e1f
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/create_data.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+echo -e "\n===== Writing data, copying data to QSFS, and checking hashes ====="
+
+ipv6=$(pulumi stack -s test | grep pub_ipv6 | tr -s " " | cut -d ' ' -f 3 | cut -d '/' -f 1)
+
+ssh -t root@$ipv6 /root/test-scripts/write_data.sh
+ssh -t root@$ipv6 /root/test-scripts/copy_to_qsfs.sh
+ssh -t root@$ipv6 /root/test-scripts/wait_all_uploads.sh
+ssh -t root@$ipv6 /root/test-scripts/check_hashes.sh
+
+# Store a copy of the hashes locally, in case we redeploy the VM
+scp "root@[$ipv6]:/root/data/md5s_original" ./
diff --git a/pulumi/tests/test-scripts-local/deploy.sh b/pulumi/tests/test-scripts-local/deploy.sh
new file mode 100755
index 0000000..0bd645d
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/deploy.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+pulumi stack init test
+
+cp vars.original.py vars.py
+pulumi up -s test -y --non-interactive
diff --git a/pulumi/tests/test-scripts-local/destroy.sh b/pulumi/tests/test-scripts-local/destroy.sh
new file mode 100755
index 0000000..336668b
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/destroy.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+# If we fail to delete the deployment, we should keep the stack around
+pulumi down -s test -y --non-interactive && pulumi stack rm -yf test
diff --git a/pulumi/tests/test-scripts-local/recover.sh b/pulumi/tests/test-scripts-local/recover.sh
new file mode 100755
index 0000000..5aac918
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/recover.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+echo -e "\n===== Running recover script on remote VM ====="
+
+ipv6=$(pulumi stack -s test | grep pub_ipv6 | tr -s " " | cut -d ' ' -f 3 | cut -d '/' -f 1)
+
+# The scripts uploaded by Pulumi won't be executable. We could fix that elsewhere
+ssh -t root@$ipv6 bash /root/scripts/recover.sh
+ssh -t root@$ipv6 mkdir /root/data
+scp ./md5s_original "root@[$ipv6]:/root/data/md5s_original"
diff --git a/pulumi/tests/test-scripts-local/redeploy.sh b/pulumi/tests/test-scripts-local/redeploy.sh
new file mode 100755
index 0000000..4c22be9
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/redeploy.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+echo -e "\n===== Redeploying with vars.new.py and issuing SIGUSR1 ====="
+
+pulumi stack init test
+
+cp vars.new.py vars.py
+pulumi up -s test -y --non-interactive
+
+ipv6=$(pulumi stack -s test | grep pub_ipv6 | tr -s " " | cut -d ' ' -f 3 | cut -d '/' -f 1)
+
+# Since we might use this script both in cases where the frontend VM is
+# replaced or not, we'll just go ahead and try to issue the SIGUSR1 even though
+# it has no effect on a fresh VM
+ssh -o StrictHostKeyChecking=accept-new -t root@$ipv6 '
+  pkill zstor -SIGUSR1
+'
diff --git a/pulumi/tests/test-scripts-local/remove_and_rebuild.sh b/pulumi/tests/test-scripts-local/remove_and_rebuild.sh
new file mode 100755
index 0000000..c1718ff
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/remove_and_rebuild.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+echo -e "\n===== Removing local data files and reconstructing from backends ====="
+
+ipv6=$(pulumi stack -s test | grep pub_ipv6 | tr -s " " | cut -d ' ' -f 3 | cut -d '/' -f 1)
+
+ssh -t root@$ipv6 'rm /data/data/zdbfs-data/*'
+ssh -t root@$ipv6 /root/test-scripts/check_hashes.sh
diff --git a/pulumi/tests/test-scripts-local/upload_remote_scripts.sh b/pulumi/tests/test-scripts-local/upload_remote_scripts.sh
new file mode 100755
index 0000000..d479efe
--- /dev/null
+++ b/pulumi/tests/test-scripts-local/upload_remote_scripts.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# We need this to run non interactively. Otherwise we'll be prompted for it
+export PULUMI_CONFIG_PASSPHRASE=""
+
+# Use first argument if provided, otherwise default to 'test'
+STACK_NAME=${1:-test}
+ipv6=$(pulumi stack -s $STACK_NAME | grep pub_ipv6 | tr -s " " | cut -d ' ' -f 3 | cut -d '/' -f 1)
+
+# Get directory of script file. This way the path to  upload is always correct
+# regardless of where the script is run from
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+echo -e "\n===== Copying remote test scripts to the VM ===="
+scp -o StrictHostKeyChecking=accept-new -r "$SCRIPT_DIR"/../test-scripts-remote/ "root@[$ipv6]:/root/test-scripts"
diff --git a/pulumi/tests/test-scripts-remote/README.md b/pulumi/tests/test-scripts-remote/README.md
new file mode 100644
index 0000000..530fd77
--- /dev/null
+++ b/pulumi/tests/test-scripts-remote/README.md
@@ -0,0 +1 @@
+This folder contains scripts that run on the remote machine during testing. They are basically about creating some data files and ensuring their integrity after being retrieved.
diff --git a/pulumi/tests/test-scripts-remote/check_hashes.sh b/pulumi/tests/test-scripts-remote/check_hashes.sh
new file mode 100755
index 0000000..9139a81
--- /dev/null
+++ b/pulumi/tests/test-scripts-remote/check_hashes.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo -e "\n===== Calculating MD5 hashes of stored files ====="
+# Calculate and MD5 hash for each file and write to new hash file. Overwrite
+# the new hashes file since we might run this multiple times
+rm -f /root/data/md5s_new
+for i in {1..10}; do
+  md5sum /mnt/qsfs/file$i.dat | cut -d " " -f 1 | tee -a /root/data/md5s_new
+done
+
+echo -e "\n===== Comparing hashes ===="
+if cmp -s /root/data/md5s_original /root/data/md5s_new; then
+  echo -e "\n===== Hashes match, success ====="
+else
+  echo -e "\n===== Hashes differ, failure ===="
+fi
diff --git a/pulumi/tests/test-scripts-remote/copy_to_qsfs.sh b/pulumi/tests/test-scripts-remote/copy_to_qsfs.sh
new file mode 100755
index 0000000..ca01b9c
--- /dev/null
+++ b/pulumi/tests/test-scripts-remote/copy_to_qsfs.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script copies the data files into the QSFS and waits for uploading to
+# complete.
+
+echo -e "\n===== Installing pv tool for transfer monitoring ====="
+apt update &> /dev/null && apt install -y pv &> /dev/null
+
+echo -e "\n===== Copying files to QSFS mount with progress monitoring ====="
+# Copy files to the qsfs mount and check speed
+for i in {1..10}; do
+    echo "Copying file$i.dat..."
+    pv -s 100m "/root/data/file$i.dat" > "/mnt/qsfs/file$i.dat"
+done
diff --git a/pulumi/tests/test-scripts-remote/wait_all_uploads.sh b/pulumi/tests/test-scripts-remote/wait_all_uploads.sh
new file mode 100755
index 0000000..ced718c
--- /dev/null
+++ b/pulumi/tests/test-scripts-remote/wait_all_uploads.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# This script waits for all files to be uploaded to zstor (the ones that are
+# expected to be uploaded, anyway).
+
+echo -e "\n===== Waiting for all data files to upload ====="
+
+wait_for_upload() {
+    while [ -z $(zstor -c /etc/zstor-default.toml check --file "$1") ]; do
+        sleep 2
+    done
+    echo $1
+}
+
+for namespace in "zdbfs-data" "zdbfs-meta"; do
+    namespace_file="/data/index/$namespace/zdb-namespace"
+    if [ -f "$namespace_file" ]; then
+        wait_for_upload $namespace_file
+    else
+        echo Namespace file missing: $namespace_file
+    fi
+
+    for type in "data" "index"; do
+        # The index directory also has the namespace file, so we exclude that by
+        # only looking for files starting with d or i
+        path_base=/data/$type/$namespace/${type:0:1}
+        # We want to check every file except for the largest sequence number, so
+        # we sort and throw away the last row. Here an ls even without -1 helps
+        # sort to work, while echo does't. Not sure why
+        for file in $(ls -1 $path_base* | sort -V | head -n -1); do
+            wait_for_upload $file
+        done
+    done
+done
diff --git a/pulumi/tests/test-scripts-remote/write_data.sh b/pulumi/tests/test-scripts-remote/write_data.sh
new file mode 100755
index 0000000..18e5812
--- /dev/null
+++ b/pulumi/tests/test-scripts-remote/write_data.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# This script generates random data files in regular storage on the VM then
+# takes the checksum of each one and stores the sums in a file. By using
+# regular storage, we get a baseline idea about write performance and also
+# ensure that our later data integrity checks compare to something totally
+# independent of the QSFS machinery.
+
+echo "===== Creating 10 test files with 100MB random data each ====="
+# Create 10 files with 100mb random data
+mkdir -p /root/data
+for i in {1..10}; do
+  echo "Creating file$i.dat..."
+  dd if=/dev/urandom of=/root/data/file$i.dat bs=1M count=10
+done
+
+echo -e "\n===== Calculating MD5 checksums of source files ====="
+# Calculate and MD5 sum for each file and write to file
+rm -f /root/data/md5s_original
+for i in {1..10}; do
+  md5sum /root/data/file$i.dat | cut -d " " -f 1 | tee -a /root/data/md5s_original
+done
diff --git a/pulumi/vars.example.py b/pulumi/vars.example.py
new file mode 100644
index 0000000..7aaeaa0
--- /dev/null
+++ b/pulumi/vars.example.py
@@ -0,0 +1,26 @@
+# These can also be specified as env vars with the same names
+MNEMONIC = "your words here"
+NETWORK = "test"
+
+# In order to run commands on the deployed VM, we need both the public and
+# private key files available. This takes the path to the private key file, and
+# there should be a matching .pub file
+SSH_KEY_PATH = "~/.ssh/id_rsa"
+
+# Node to deploy VM on. Can overlap with Zdb nodes or not, doesn't matter
+VM_NODE = 5
+
+# Nodes to deploy Zdbs on
+META_NODES = [1, 3, 5, 8]
+DATA_NODES = [1, 3, 5, 8]
+
+# Size of each data backend Zdb in GB
+DATA_SIZE = 1
+
+# Network used to connect to the backend zdbs
+# ZDB_CONNECTION = "mycelium"
+ZDB_CONNECTION = "ipv6"
+
+# Network used for SSH connection
+# SSH_CONNECTION = "mycelium"
+SSH_CONNECTION = "ipv6"
diff --git a/pulumi/zinit/node-exporter.yaml b/pulumi/zinit/node-exporter.yaml
new file mode 100644
index 0000000..1a466e1
--- /dev/null
+++ b/pulumi/zinit/node-exporter.yaml
@@ -0,0 +1 @@
+exec: prometheus-node-exporter
diff --git a/pulumi/zinit/prometheus-pushgateway.yaml b/pulumi/zinit/prometheus-pushgateway.yaml
new file mode 100644
index 0000000..092fc4d
--- /dev/null
+++ b/pulumi/zinit/prometheus-pushgateway.yaml
@@ -0,0 +1 @@
+exec: prometheus-pushgateway --persistence.file=/var/lib/prometheus/pushgateway-persistence.data
diff --git a/pulumi/zinit/prometheus.yaml b/pulumi/zinit/prometheus.yaml
new file mode 100644
index 0000000..3e07ecc
--- /dev/null
+++ b/pulumi/zinit/prometheus.yaml
@@ -0,0 +1 @@
+exec: prometheus --config.file=/etc/prometheus.yaml
diff --git a/pulumi/zinit/zdb.yaml b/pulumi/zinit/zdb.yaml
new file mode 100644
index 0000000..34ae96c
--- /dev/null
+++ b/pulumi/zinit/zdb.yaml
@@ -0,0 +1,10 @@
+exec: |
+  /usr/local/bin/zdb \
+    --index /data/index \
+    --data /data/data \
+    --logfile /var/log/zdb.log \
+    --datasize 67108864 \
+    --hook /usr/local/bin/zdb-hook.sh \
+    --rotate 900
+shutdown_timeout: 60
+after: [zstor]
diff --git a/pulumi/zinit/zdbfs.yaml b/pulumi/zinit/zdbfs.yaml
new file mode 100644
index 0000000..c999d7d
--- /dev/null
+++ b/pulumi/zinit/zdbfs.yaml
@@ -0,0 +1,2 @@
+exec: /usr/local/bin/zdbfs /mnt/qsfs -o autons
+after: [zdb]
diff --git a/pulumi/zinit/zstor.yaml b/pulumi/zinit/zstor.yaml
new file mode 100644
index 0000000..0f1a98f
--- /dev/null
+++ b/pulumi/zinit/zstor.yaml
@@ -0,0 +1,6 @@
+exec: |
+  /bin/zstor \
+  -c /etc/zstor-default.toml \
+  --log_file /var/log/zstor.log \
+  monitor
+shutdown_timeout: 300
diff --git a/pulumi/zstor_config.base.example.toml b/pulumi/zstor_config.base.example.toml
new file mode 100644
index 0000000..f088569
--- /dev/null
+++ b/pulumi/zstor_config.base.example.toml
@@ -0,0 +1,20 @@
+minimal_shards = 2
+expected_shards = 4
+redundant_groups = 0
+redundant_nodes = 0
+root = "/"
+zdbfs_mountpoint = "/mnt/qsfs"
+socket = "/tmp/zstor.sock"
+prometheus_port = 9200
+zdb_data_dir_path = "/data/data/zdbfs-data/"
+max_zdb_data_dir_size = 2560
+
+[compression]
+algorithm = "snappy"
+
+[meta]
+type = "zdb"
+
+[meta.config]
+prefix = "zstor-meta"
+