From ade09459538d04b29e39d9c99950691661986a74 Mon Sep 17 00:00:00 2001 From: Scott Yeager Date: Tue, 25 Mar 2025 17:37:46 -0700 Subject: [PATCH] Update docs --- config/zstor-sample-local.toml | 58 -- docs/01_overview.md | 19 + docs/02_planning.md | 75 +++ docs/03_deployment_options.md | 21 + docs/04_deploy_tfcmd.md | 554 ++++++++++++++++++ docs/05_deploy_pulumi.md | 111 ++++ docs/06_operating.md | 110 ++++ ...oubleshooting.md => 07_troubleshooting.md} | 10 +- docs/{deployment.md => archive/diagram.md} | 0 docs/encryption.md | 10 - docs/example_zstor_config.toml | 107 ---- docs/manual_setup.md | 70 --- docs/operations.md | 23 - 13 files changed, 896 insertions(+), 272 deletions(-) delete mode 100644 config/zstor-sample-local.toml create mode 100644 docs/01_overview.md create mode 100644 docs/02_planning.md create mode 100644 docs/03_deployment_options.md create mode 100644 docs/04_deploy_tfcmd.md create mode 100644 docs/05_deploy_pulumi.md create mode 100644 docs/06_operating.md rename docs/{troubleshooting.md => 07_troubleshooting.md} (89%) rename docs/{deployment.md => archive/diagram.md} (100%) delete mode 100644 docs/encryption.md delete mode 100644 docs/example_zstor_config.toml delete mode 100644 docs/manual_setup.md delete mode 100644 docs/operations.md diff --git a/config/zstor-sample-local.toml b/config/zstor-sample-local.toml deleted file mode 100644 index f77d740..0000000 --- a/config/zstor-sample-local.toml +++ /dev/null @@ -1,58 +0,0 @@ -data_shards = 1 -parity_shards = 1 -redundant_groups = 0 -redundant_nodes = 0 -root = "/virtualroot" -socket = "/tmp/zstor.sock" -network = "Main" -wallet_secret = "SABDZC5WIRSR4A6T5JOVOYXTXKSGE6LBX7GD4CMNVHQK4WN5MEQNGRKB" -identity_id = 0 -identity_mnemonic = "shop moral play negative liar lucky hub stock merge sea treat spare spray december media setup mean narrow combine merge mixture arm boy stem" -horizon_url = "https://my.horizon.server" -prometheus_port = 9100 -zdb_data_dir_path = "/tmp/0-db/data" -max_zdb_data_dir_size = 256000 -zdbfs_mountpoint = "/somewhere" - -[encryption] -algorithm = "AES" -key = "0000000000000000000000000000000000000000000000000000000000000000" - -[compression] -algorithm = "snappy" - -[meta] -type = "zdb" - -[meta.config] -prefix = "debug-prefix" - -[meta.config.encryption] -algorithm = "AES" -key = "0101010101010101010101010101010101010101010101010101010101010101" - -[[meta.config.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-meta-1" - -[[meta.config.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-meta-2" - -[[meta.config.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-meta-3" - -[[meta.config.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-meta-4" - -[[groups]] -[[groups.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-backend-1" - -[[groups.backends]] -address = "127.0.0.1:9901" -namespace = "zstor-backend-2" - diff --git a/docs/01_overview.md b/docs/01_overview.md new file mode 100644 index 0000000..76d12b6 --- /dev/null +++ b/docs/01_overview.md @@ -0,0 +1,19 @@ +## Introduction + +Quantum Safe Storage (also known as Quantum Safe Filesystem or QSFS) is a distributed data store that is designed to provide resilience, security, and good performance. It is implemented on the frontend as a FUSE filesystem (Filesystem in Userspace) that can be mounted on any Linux machine. All files written to this filesystem are then dispersed among a configurable number of backends, such that failure of X nodes or Y groups of nodes can be tolerated without losing any data. + +The system can support petabytes of total capacity, and the frontend serves as a cache of configurable size. Data blocks are encrypted and dispersed using forward looking error correcting codes (FLECC). Once stored in the backends, blocks can be freed from the frontend to make room for fresh data once the cache is full. When that data is needed again, it is reconstructed and decrypted on the fly. + +Thanks to the methods used, not even a quantum computer capable of breaking encryption can decode data stored in the backends. + +## Components + +There are three main components comprising QSFS. These components are: + +- [0-db-fs](https://github.com/threefoldtech/0-db-fs) (also known as zdbfs) is what creates the FUSE mount (the actual user facing filesystem). This component is not aware of the backend operations like encryption and FLECC. Its main job is to expose the FUSE filesystem and store its data in a local zdb instance. +- [0-db](https://github.com/threefoldtech/0-db) (also known as zdb) is used both for the local cache db and also for the backend data stores. Zdb is a fast and efficient append-only key value database. +- [0-stor](https://github.com/threefoldtech/0-stor_v2) (also known as zstor) is responsible for the encryption and FLECC operations on data blocks and database indexes, storing them among the configured backends. + +Each component is also capable of independent operation. For example, zstor can be used to store individual files of any kind. Zdb is a general purpose key value store compatible with a subset of Redis operations, and zdbfs can be used without any offloading of data to backends. + +More information about these projects can be found in the repositories linked above. diff --git a/docs/02_planning.md b/docs/02_planning.md new file mode 100644 index 0000000..48d031a --- /dev/null +++ b/docs/02_planning.md @@ -0,0 +1,75 @@ +## Introduction + +In this section, we cover the planning and selection process of QSFS backends. + +The backend storage we will consider here is an HDD-based zdb capacity provisioned on the ThreeFold Grid. While backends must always use zdb, it is possible to host these on other systems or storage media. That being said, such configurations will not be covered here. + +## Values Overview + +When data is pushed into the backends by zstor, it is done so according to a few configuration values. In a general sense, these values determine the amount of overhead that is used to provide redundancy and fault tolerance, but they can also affect performance. + +Here's an overview of the most important values to consider when planning the backends for your QSFS deployment: + +* **Minimal shards** - The minimum number of data shards needed to recreated a given data block. +* **Expected shards** - The total amount of shards which are generated when the data is encoded. +* **Disposable shards** - How many shards can be lost without losing any data. While this is not configured directly, it is derived from the previous two values: + * `disposable shards = expected shards - minimal shards` +* **Groups** - backends can be grouped together into logical groups. Typically, each group would represent a physical site. +* **Redundant groups** - How many groups can be lost while keeping the data available. +* **Redundant nodes** - How many nodes can be lost in each group while keeping the data available in that group. +* **Backend size** - The size of each backend. In this guide, we will use gigabytes. + + +For planning purposes, it can be helpful to know how much data can be stored in a given configuration. Here are a few example calculations for a basic single group setup with minimal shards of 16 and expected shards of 20: + +``` +disposable shards = expected shards - minimal shards = 20 - 16 = 4 + +overhead = disposable shards / minimal shards = 4 / 16 = 25% = 0.25 +``` + +Now let's say that we provision 20 backends with 1 GB of storage each: + +``` +total backend storage = 20 * 1 GB = 20 GB + +overhead = 20 GB * 0.25 = 5 GB + +usable capacity = 20 GB - 5 GB = 15 GB +``` + +When using redundant groups, there is additional overhead because each group must store at least the minimal shards. The exact figures depend on the configuration. In the case that each group holds the expected shards, then the total usable capacity is that of a single group (the smallest if the groups have different total storage sizes). + +## Zstor Metadata + +In addition to the data backends, zstor also requires exactly four metadata backends. This value is hard-coded for now, as is the quantity of disposable metadata shards, which is two. That means that two of four metadata backends can be lost while the operation of the system can continue normally. If three or all four of the metadata backends are unreachable, then no data can be retrieved by zstor. In that case, only any data cached in the frontend would remain available. + +Therefore, to tolerate the failure of a site, the metadata nodes would need to be split among at least two sites. Metadata and data shards can be stored on the same nodes, but the zdbs should be provisioned separately in any case. + +## Finding backend Nodes + +Since we require HDD storage for backends, this will be the primary constraint when searching for host nodes. One way to do this is to use the [node finder](https://dashboard.grid.tf/#/deploy/node-finder) in the ThreeFold Dashboard and use a minimum HDD capacity filter. + +Grouping nodes according to farm can be a strategy for ensuring the resulting system can tolerate site failures. We should note however that a given farm can contain nodes in different physical locations, and different farms can have their nodes in the same location. While the longitude and latitude data in the Dashboard is not precise, it's the best indication of whether nodes are in the same place or at least if they are connected to the same ISP regional hub. + +## Backend Networking + +With the release of ThreeFold Grid 3.14, zdbs provisioned through Zero OS have three networking options: IPv6, Yggdrasil (Planetary Network), and Mycelium. For this guide, we will stick to Mycelium because it is available on every node and thus simplifies the deployment process. + +For more information about the tradeoffs involved in using IPv6 for zdb networking, see the [section below](#ipv6-as-a-zdb-networking-option). + +## Additional backend Planning Considerations + +To take advantage of the redundancy properties of QSFS, you will need to find at least as many backend nodes as your planned expected shards count. Placing multiple backends on the same node invalidates the calculations and reasoning about fault tolerance. Therefore, it is not recommended. Optionally, you can also find four additional nodes for metadata storage, if for some reason you don't want to mix data and metadata storage nodes. + +It is also possible to provision more data backends than the expected shards amount. In this case, some nodes will not store any shards for each given data block. Data is still only guaranteed to be durable for the failure of nodes equal to the number of disposable shards, though in practice the availability of a given data block will depend only on the availability of the minimal shard count of nodes where data for that block is actually stored. + +We will cover replacing backends in the final section of this guide. The reason for this could be to replace backends that have failed or to increase the storage amount. For now, it might be useful to note a few potential replacement nodes that can be swapped in later for any failed backends. + +## IPv6 as a Zdb Networking Option + +Using IPv6 for zdb connectivity could have some advantages. Because the data sent to backends is already quantum safe, there's no need to encrypt it again for transport as Mycelium does for all traffic. It's also possible that IPv6 provides a much more direct route, especially in the case that the frontend is in the same LAN as some or all of the backends. All Mycelium traffic must pass over a publicly reachable Mycelium node, and that could mean routing traffic over another physical location even when the machines communicating are side by side. + +Using IPv6 has a few downsides for this use case. It is not every node that supports IPv6 at all, and among those that do, not all of them are publicly reachable. There are no simple ways to find which nodes or farms provide publicly reachable IPv6 addresses. It's also possible that IPv6 addresses are reassigned by the farmer's ISP. While some farmers certainly have stable IPv6 blocks, it is again not possible to find this information in any simple way. + +Due to these difficulties, the details of how to provision and use backend zdbs with IPv6 isn't covered in this guide. It can be accomplished, however, with some small changes to the process shown. diff --git a/docs/03_deployment_options.md b/docs/03_deployment_options.md new file mode 100644 index 0000000..2ea3dce --- /dev/null +++ b/docs/03_deployment_options.md @@ -0,0 +1,21 @@ +There are a variety of ways to deploy the system: + +1. As a primitive provided by Zos +2. Using cli tools and scripts +3. Via a Pulumi based deployer found in this repo + +These have various tradeoffs that are explored below. + +## Zos Primitive + +The Zos primitive provides a convenient way to deploy Quantum Safe Storage but it isn't very flexible. It's not possible to access the zstor process and certain functions like recovering to a new frontend machine are not available. + +This method could be useful for basic testing. How to deploy the primitive version using Terraform is [covered in the manual](https://manual.grid.tf/documentation/system_administrators/terraform/resources/terraform_qsfs.html). + +## Cli Tools and Scripts + +Using ThreeFold's `tfcmd` cli tool, it's possible to deploy all elements needed for Quantum Safe Storage. This method offers the most flexibility and control over the deployment process. An example for how to do this is included in a later section of these docs. + +## Pulumi Deployer + +The Pulumi deployer is [included](https://github.com/threefoldtech/quantum-storage/tree/master/pulumi) in this repo. It offers a largely automated process that covers advanced use cases like replacing failed backends and recovering to a new frontend machine. diff --git a/docs/04_deploy_tfcmd.md b/docs/04_deploy_tfcmd.md new file mode 100644 index 0000000..b3dbd56 --- /dev/null +++ b/docs/04_deploy_tfcmd.md @@ -0,0 +1,554 @@ +## Introduction + +This section covers the manual deployment steps for QSFS, including both the backend zdbs and an Ubuntu VM to host the frontend. It is also possible to host the frontend on pretty much any other Linux machine as long as FUSE is available. At this point you should already have node IDs selected for your backend zdbs. If not, please see the planning section above for more details. + +While there are several ways to deploy zdbs on the ThreeFold Grid, including using Terraform and custom code using one of the SDKs, this guide shows how to to do this with our simple command line tool, tfcmd. This tool is available both for Linux and MacOS. For best security of the seed phrase used to create the deployments, it is recommended to run tfcmd on your local machine. However, it is also possible to use it inside the VM deployed for the QSFS frontend. + +## Deployment Prerequisites + +Before proceeding, we assume that you have an activated TFChain account funded with TFT and ready to create deployments. If not, see the [getting started](https://manual.grid.tf/documentation/system_administrators/getstarted/tfgrid3_getstarted.html) section of the manual for more details. + +## Deploy Frontend VM + +Both full and micro VMs work fine for the QSFS frontend. + +There are a few considerations to keep in mind for this VM: + +- QSFS will consume more CPU and RAM as load increases. +- 1 vCPU and 2 GB of RAM can work for light loads, but at least an additional vCPU will be helpful for heavier loads. +- The SSD capacity of the VM is the maximum available frontend cache size for QSFS. +- You should enable Mycelium networking, as this will be required to connect to the zdbs. + +You can deploy this VM using the [Dashboard](https://dashboard.grid.tf/#/deploy/virtual-machines/) or via the various other methods described in the ThreeFold Manual. + +If you plan to use the frontend VM to run tfcmd and deploy the zdbs, then connect to the VM via SSH now and run all the following commands on the VM. + +## Deploy Backend Zdbs + +Now we will deploy the backend zdbs using tfcmd. To assist with creating multiple deployments efficiently, some short scripts will be shown below. These scripts have been tested in bash and might not work in other shells. At the same time, we will also create our zstor config file, which must contain information about the backends. + +### Prerequisites + +We will use `wget`, `jq`, and `openssl`. Make sure that they are properly installed on the system you will be using. On Ubuntu, for example, you can use this command: + +```sh +apt update && apt install -y wget jq openssl +``` + +### Set Your EDITOR + +In the sections below, we'll be creating a number of config files. Set your preferred editor now to facilitate copying and pasting the commands. For example: + +```sh +EDITOR=nano +``` + +### Install Tfcmd + +Here we will fetch the `tfcmd` binary and install it locally. It is released as part of `tfgrid-sdk-go`. You can find the latest releases [here](https://github.com/threefoldtech/tfgrid-sdk-go/releases). Choose the correct download link for your platform. + +For example, to install version 15.10 on a x64 Linux machine, you would run the following: + +``` +wget -O tfgrid-sdk-go.tar.gz https://github.com/threefoldtech/tfgrid-sdk-go/releases/download/v0.15.10/tfgrid-sdk-go_Linux_x86_64.tar.gz +tar -C /usr/local/bin -xf tfgrid-sdk-go.tar.gz tfcmd +chmod +x /usr/local/bin/tfcmd +rm tfgrid-sdk-go.tar.gz +``` + +To test that it worked, try logging in. You'll need to complete this step before creating any deployments: + +```sh +tfcmd login +``` + + +### Create Stub Zstor Config + +Next we will create a stub of the zstor config file. This will contain all of the information that needs to be filled manually. The final sections with the backend info will be filled automatically using scripts. + +Open a file `zstor-default.toml` in a text editor and paste in the template below. For example: + +```sh +$EDITOR zstor-default.toml +``` + +Initial contents of the file: + +``` +minimal_shards = 16 +expected_shards = 20 +redundant_groups = 0 +redundant_nodes = 0 +zdbfs_mountpoint = "/mnt/qsfs" +socket = "/var/run/zstor.sock" +prometheus_port = 9100 +zdb_data_dir_path = "/data/data/zdbfs-data" +max_zdb_data_dir_size = 25600 + +[encryption] +algorithm = "AES" +key = "Write your key here" + +[compression] +algorithm = "snappy" + +[meta] +type = "zdb" + +[meta.config] +prefix = "qsfs-meta" + +[meta.config.encryption] +algorithm = "AES" +key = "write your key here" +``` + +Make sure to edit the file as needed. You can change the minimal and expected shards according to your own plan. Another important value is `max_zdb_data_dir_size`, which is how large the cache is allowed to grow before data blocks are removed. This value is given in MiB. Therefore, the example shown is 25GiB. + +The `zdbfs_mountpoint` can also be adjusted to wherever you want to mount the filesystem, and that could be any location of your choosing. This configuration value does not actually control the placement of the mount, however, it is just used by zstor for monitoring purposes. The actual mounting will happen later. Likewise `zdb_data_dir_path` should be updated if you want to place the zdb data directory somewhere else. + +It is also necessary to fill in the encryption keys with your own. You can use the same or different keys for data and metadata, at your own preference. The key must be 32 bytes in hex format. Here's an example of how to generate a key in this format: + +```sh +openssl rand -hex 32 +``` + +Once this is done, take the output and insert it into the file in the indicated locations. + +### Prepare for Zdb Deployment + +Now we will deploy the zdbs and write their details into the config file in the proper format. Before proceeding, we'll set a few shell variables that will be used in the deployment scripts: + +```sh +CONFIG=zstor-default.toml +PASSWORD=$(openssl rand -base64 18) +METADATA_NODES="1 2 3 4" +BACKEND_NODES="1 2 3 4 5 6 7 8" +BACKEND_SIZE=1 +``` + +This will generate a strong random password that will be used to secure each zdb. You can replace the code that generates the password with your own password if you wish. For now, don't worry about having to save the randomly generated password. It will get written to the config file and you can take note of it later. + + +For the metadata and backend nodes, replace the example values with the node IDs you selected before. Set your desired backend size too, which is specified in gigabtyes. For more details on selection of backend nodes, and how many of each type to specify, see the previous section on planning a deployment. + + +### Deploy Metadata Zdbs + +Here is an example script to deploy the metadata zdbs based on the variables set above. This uses a fixed size of 1 GB, which is the minimum when deploying via tfcmd and should be plenty for storing all of the metadata ever needed by a typical QSFS deployment. Metadata zdbs run in `user` mode. + +```sh +for node in $METADATA_NODES; do + name=node${node}meta + tfcmd deploy zdb --mode user --node $node -n $name --size 1 --password $PASSWORD +done +``` + +It's possible that some of your chosen nodes don't respond properly at deployment time and need to be replaced with other nodes. In that case, just replace the variable with the new node IDs like this: + +```sh +# Example: Node 3 wasn't working, replace it with node 5 +METADATA_NODES="1 2 4 5" +``` + +Then you can just run the original deployment script loop again. Any zdb deployments that would be duplicated will be detected by `tfcmd` and skipped over. + +### Cancel Metadata Zdbs + +If you need to cancel the metadata zdb deployments, use this script: + +```sh +for node in $METADATA_NODES; do + name=node${node}meta + tfcmd cancel $name +done +``` + +### Write Metadata Zdbs Config + +Next we will write the configuration data from the deployed zdbs into the config file. The script as shown is for IPv6 connections. If you are using Yggdrasil, replace `.Zdbs[0].ips[0]` with `.Zdbs[0].ips[1]` + +```sh +# Wait about five seconds before doing the next step to make sure data is available. Only needed if running immediately after the deployment step in a single script +sleep 5 + +# Write out the config sections for the metadata zdbs +for node in $METADATA_NODES; do + name=node${node}meta + echo Fetching and writing config for $name + json=$(tfcmd get zdb $name 2>&1 | tail -n +3 | sed $'s/\e\[0m//') + ip=$(echo $json | jq .Zdbs[0].ips[-1] | tr -d \") + port=$(echo $json | jq .Zdbs[0].port) + namespace=$(echo $json | jq .Zdbs[0].namespace) + password=$(echo $json | jq .Zdbs[0].password) + + echo \# Node $node >> $CONFIG + echo [[meta.config.backends]] >> $CONFIG + echo address = \"\[$ip\]:$port\" >> $CONFIG + echo namespace = $namespace >> $CONFIG + echo password = $password >> $CONFIG + echo >> $CONFIG +done +``` + +Once that has completed, you can check inside your `zstor-default.toml` file to see the result. There should be four sections that look like this: + +``` +# Node 1 +[[meta.config.backends]] +address = "[2a02:1802:5e:0:c11:7dff:fe8e:83bb]:9900" +namespace = "18-532404-node1meta0" +password = "Your password" +``` + +Sometimes `tfcmd` fails to fetch deployment information from one or more nodes. In that case, you might see one block with blank fields. You can try generating those sections again by changing the script to only target those node IDs. + +Here's an example to retry nodes 2 and 3: + +```sh +# Example with nodes 2 and 3. Write out the config sections for the metadata zdbs +for node in 2 3; do + # The rest of the script is the same + # ... +``` + +Then check the file again. Make sure that all the metadata nodes you specified have a populated config section and that result of any failed attempts are deleted. + +### Deploy Data Zdbs + +This process is very similar to the deployment of the metadata backends, with a few small changes to the scripts. We use the variable `BACKEND_SIZE` we defined above and `seq` mode this time. + +To deploy: + +```sh +for node in $BACKEND_NODES; do + name=node${node}backend + tfcmd deploy zdb --mode seq --node $node -n $name --size $BACKEND_SIZE --password $PASSWORD +done +``` + +As before, you might need to replace some node IDs and try again: + +```sh +# Example: Node 3 wasn't working, replace it with node 9 +BACKEND_NODES="1 2 4 5 6 7 8 9" +``` + +Then run the deployment loop again. + +### Cancel Metadata Zdbs + +Likewise, if you need to cancel the data zdb deployments, use this script: + +```sh +for node in $BACKEND_NODES; do + name=node${node}backend + tfcmd cancel $name +done +``` + +### Write Data Zdbs Config + +To write the config for the data zdbs, use the following script: + +```sh +# Ditto, need to wait +sleep 5 + +echo [[groups]] >> $CONFIG +for node in $BACKEND_NODES; do + name=node${node}backend + echo Fetching and writing config for $name + json=$(tfcmd get zdb $name 2>&1 | tail -n +3 | sed $'s/\e\[0m//') + ip=$(echo $json | jq .Zdbs[0].ips[-1] | tr -d \") + port=$(echo $json | jq .Zdbs[0].port) + namespace=$(echo $json | jq .Zdbs[0].namespace) + password=$(echo $json | jq .Zdbs[0].password) + + echo \# Node $node >> $CONFIG + echo [[groups.backends]] >> $CONFIG + echo address = \"\[$ip\]:$port\" >> $CONFIG + echo namespace = $namespace >> $CONFIG + echo password = $password >> $CONFIG + echo >> $CONFIG +done +``` + +Notice this time that the data backends have an extra line `[[groups]]` separating them from the top of the file. This script just creates a single group. If you want to use more groups, add more groups lines to separate the backends in each group. + +As before, check the output for any failures to retrieve data. You can retry them in the same way: + +```sh +# Note that we skipped the line with [[groups]] +for node in $BACKEND_NODES; do + # The rest of the script is the same + # ... +``` + +Once every data backend has a valid entry in the config file, we are done with this section of the deployment. + +### Storing Zstor Config + +The `zstor-default.toml` file contains sensitive information that is sufficient to recover and decrypt all of the data stored in your QSFS. Needless to say, you should keep the contents of this file secure. + +If your frontend machine is lost for any reason, the zstor config file can be used to recover the data. On the other hand, if the contents of this file are lost, the data in the backends can never be recovered. + +*Consider storing the entire contents of your `zstor-default.toml` file in a durable and secure data store like a password manager.* + +## Frontend System Prep + +At this point, we are ready to proceed the setup of the frontend system. We'll demonstrate all necessary commands to do this in any Linux system that already has `wget` installed, FUSE enabled in the kernel, and a Mycelium connectivity. If you deployed a VM on the ThreeFold Grid according to the instructions above, then these requirements are already met. + +### Install Binaries + +The three binary executables needed to operate QSFS are provided in statically compiled form with no dependencies. You can download them from GitHub according to the links on each project's release page: + +- 0-db-fs: [https://github.com/threefoldtech/0-db-fs/releases](https://github.com/threefoldtech/0-db-fs/releases) +- 0-db: [https://github.com/threefoldtech/0-db/releases](https://github.com/threefoldtech/0-db/releases) +- 0-stor: [https://github.com/threefoldtech/0-stor_v2/releases](https://github.com/threefoldtech/0-stor_v2/releases) + + +Here is an example with the latest versions of each component at the time of publishing this guide. We will also download a hook script that is the final needed component: + +``` +wget -O /usr/local/bin/zdbfs https://github.com/threefoldtech/0-db-fs/releases/download/v0.1.11/zdbfs-0.1.11-amd64-linux-static +wget -O /usr/local/bin/zdb https://github.com/threefoldtech/0-db/releases/download/v2.0.8/zdb-2.0.8-linux-amd64-static +wget -O /bin/zstor https://github.com/threefoldtech/0-stor_v2/releases/download/v0.4.0/zstor_v2-x86_64-linux-musl +wget -O /usr/local/bin/zdb-hook.sh https://raw.githubusercontent.com/threefoldtech/quantum-storage/master/lib/zdb-hook.sh + +# Make them all executable +chmod +x /usr/local/bin/* /bin/zstor +``` + +One note here is that the name and location of the `zstor` executable must match what is shown here for the hook script to work properly. + +### Directories + +Two directories will be needed for QSFS operation. First is the QSFS mount point and second is the data directory for the local zdb. + +The locations shown below are examples and you can change them if you wish. However if you choose to use different locations for either the mount point or the data folder, don't forget to update them accordingly in the zstor config file and also substitute them throughout the guide below. + +Make sure both directories exist like this: + +```sh +mkdir -p /mnt/qsfs +mkdir -p /data +``` + +### Set EDITOR + +Again, make sure your preferred editor is set if you want easy copy and paste: + +```sh +EDITOR=nano +``` + +### Zstor Config + +Now copy your `zstor-default.toml` to the `/etc` folder inside the VM. You could use `scp` for this, or copy and paste the contents into a new file on the frontend machine: + +```sh +$EDITOR /etc/zstor-default.toml +``` + +## Frontend Services Setup + +Now we will demonstrate how to configure each of the three QSFS components as a system service managed by a process manager. There are two different options here. First is systemd, which comes standard in most Linux distributions including ThreeFold full VMs. The second will be zinit, which is the process manager provided in our official micro VM images where systemd is not in use. Example config files and commands will be shown for each case. + +### Note on Systemd Service Restarts + +The default behavior of systemd is to restart services fast if they exit unexpectedly but also to quickly give up on restarting a service that continues to exit. + +To ensure that systemd always tries to restart our services, no matter how many failures occur, `StartLimitIntervalSec` is set to zero in the examples below to disable the restart limit. However, in the event of a crash loop this can put a fair amount of strain on the CPU. + +While crash loops are by no means expected, you may want to increase the time that systemd waits before trying to restart the services. Therefore, the default value of `RestartSec` is also written in case you want to change it. The tradeoff is that in the event of a recoverable failure, a longer timeout means that the service is offline for more time. + +### Zstor + +The example service config files for zstor are shown below. There is another value you might want to tune in these files, which is the timeout for stopping the service. For systemd, this is called `TimeoutStopSec` and for zinit it is called `shutdown_timeout`. + +In both cases, these values determine how long the process manager will wait between asking the process to exit on its own and issuing a kill signal in case the process does not respond. + +For zstor, this timeout represents the max time available to write data to the backends when the service is stopped, such as during a graceful system shutdown in the case of systemd. For the examples a fairly generous five minutes is used, but a longer timeout might be safer. + +#### Zstor Systemd + +```sh +$EDITOR /etc/systemd/system/zstor.service +``` + +``` +[Unit] +Wants=network.target +After=network.target +StartLimitIntervalSec=0 + +[Service] +ProtectHome=true +ProtectSystem=true +ReadWritePaths=/data /var/log +ExecStart=/bin/zstor \ + --log_file /var/log/zstor.log \ + -c /etc/zstor-default.toml \ + monitor +Restart=always +RestartSec=100ms +TimeoutStopSec=5m + +[Install] +WantedBy=multi-user.target +``` + +#### Zstor Zinit + +```sh +$EDITOR /etc/zinit/zstor.yaml +``` + +```yaml +exec: /bin/zstor \ + --log_file /var/log/zstor.log \ + -c /etc/zstor-default.toml \ + monitor +shutdown_timeout: 300 +``` + +### Zdb + +Next is zdb. There are two arguments here that might be of interest for tuning. The first is `--datasize`, which is the maximum size of data blocks, in bytes. Here use 64MiB. + +The other argument to consider is `--rotate`, which is the time at which incomplete data blocks are closed and backed up. This value is in seconds, so the example is 15 minutes. Reducing this time can help reduce the chance of data loss if the frontend is lost, but it will also result in more data fragmentation which can impact performance + +In this case, we use a shorter `TimeoutStopSec`, to give some time for zdb to flush remaining data to disk, but with the assumption that this happens much more quickly than zstor's network based operations. + +#### Zdb Systemd + +```sh +$EDITOR /etc/systemd/system/zdb.service +``` + +``` +[Unit] +Wants=network.target zstor.service +After=network.target zstor.service + +[Service] +ProtectHome=true +ProtectSystem=true +ReadWritePaths=/data /var/log +ExecStart=/usr/local/bin/zdb \ + --index /data/index \ + --data /data/data \ + --logfile /var/log/zdb.log \ + --datasize 67108864 \ + --hook /usr/local/bin/zdb-hook.sh \ + --rotate 900 +Restart=always +RestartSec=5 +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target +``` + +#### Zdb Zinit + +```sh +$EDITOR /etc/zinit/zdb.yaml +``` + +```yaml +exec: /usr/local/bin/zdb \ + --index /data/index \ + --data /data/data \ + --logfile /var/log/zdb.log \ + --datasize 67108864 \ + --hook /usr/local/bin/zdb-hook.sh \ + --rotate 900 +shutdown_timeout: 60 +after: zstor +``` + +### Zdbfs + +And finally, zdbfs. The only option to configure here is the mount point. For this example we use `/mnt/qsfs`. Remember that if you are using a different mount point, it's best to also update it in the zstor config file. + +In the case of zdbfs, the stop timeouts are less relevant. When zdbfs receives `TERM` it will exit regardless of any ongoing writes, and those write operations will encounter an error. There is a final flush to the zdb, but this happens very quickly when the zdb is running on the same machine so five seconds should be more than enough. + +Note that upon subsequent starts when using the `autons` option, zdbfs will give some non fatal errors about not being able to create zdb namespaces. These are really just informational messages that can be ignored. + +#### Zdbfs Systemd + +``` +$EDITOR /etc/systemd/system/zdbfs.service +``` + +``` +[Unit] +Wants=network.target zdb.service +After=network.target zdb.service + +[Service] +ExecStart=/usr/local/bin/zdbfs /mnt/qsfs -o autons +Restart=always +RestartSec=5 +TimeoutStopSec=5 + +[Install] +WantedBy=multi-user.target +``` + +#### Zdbfs Zinit + +```sh +$EDITOR /etc/zinit/zdbfs.yaml +``` + +In the case of zdbfs, the shutdown timeout is less relevant. When zdbfs receives `TERM` it will exit regardless of any ongoing writes, and those writes operations will encounter an error. There is a final flush to the zdb, but this happens very quickly when the zdb is running on the same machine so five seconds should be more than enough. + +```yaml +exec: /usr/local/bin/zdbfs /mnt/qsfs -o autons +after: zdb +``` + +### Start the Services + +Next we will start up all the services and then check that everything is working properly. + +#### Systemd + +```sh +systemctl daemon-reload +systemctl enable --now zstor zdb zdbfs +``` + +#### Zinit + +```sh +zinit monitor zdbfs +zinit monitor zdb +zinit monitor zstor +``` + +### Check Operation + +Check that zstor is working well: + +```sh +zstor --log_file ~/zstor.log -c /etc/zstor-default.toml status +``` + +It should show info on each backend and ideally they should all be reachable. + +You can also check that zdbfs is mounted: + +```sh +df +``` + +There should be an entry with type `zdbfs` mounted at your specified mount point. Now any files you write to the mount point will be encoded and uploaded to the backends, either when a zdb data block fills or when the rotate time has passed. + +## Conclusion + +Deployment of QSFS is now complete. In the next section, we'll cover concerns regarding the ongoing operation of a QSFS system, including how to recover from backend failures. diff --git a/docs/05_deploy_pulumi.md b/docs/05_deploy_pulumi.md new file mode 100644 index 0000000..c7ce24b --- /dev/null +++ b/docs/05_deploy_pulumi.md @@ -0,0 +1,111 @@ +# Deploy QSFS with Pulumi + +This repo includes a Pulumi deployment script in Python that fully automates the setup of a QSFS instance. The following steps are required to use this script: + +1. Install Pulumi and Python on your system +2. Use Pip to install the Python dependencies +3. Copy and edit vars.py and zstor_config.base.toml + +Only Linux and MacOS are supported. If you run Windows, I'd recommend equipping yourself with a WSL environment. + +## Install Pulumi and Python + +We won't cover the details here. Probably your system already has `python3`. + +For Pulumi, check here: https://www.pulumi.com/docs/iac/download-install/ + +## Clone the repo + +If you didn't already, go ahead and clone this repo to your local machine: + +``` +git clone https://github.com/threefoldtech/quantum-storage.git +cd quantum-storage/pulumi +``` + +## Install Python dependencies + +We need some Python packages to make this work. Using a venv is recommended. + +``` +python -m venv .venv +source .venv/bin/activate +pip install pulumi pulumi_random pulumi_command pulumi_threefold +``` + +## Prep config + +Two config files are needed. Examples are included here. Copy the examples to the expected paths, then edit the files according to your needs. + +``` +cp vars.example.py vars.py +cp zstor_config.base.example.toml zstor_config.base.toml + +$EDITOR vars.py +$EDITOR zstor_config.base.toml +``` + +## Monitoring + +This deployment can help to automate the collection of monitoring data using Prometheus. In particular, it helps facilitate a setup whereby a Prometheus instance in the frontend machine scrapes the metrics and performs a remote write to an external Prometheus instance. + +To use the feature, simple provide a `prometheus.yaml` file. If present, the deployment script will install Prometheus, upload the config file, and run Prometheus with that config as a zinit service. + +There's an example Prometheus config included that you can use as a starting point: + +``` +cp prometheus.example.yaml prometheus.yaml +$EDITOR prometheus.yaml +``` + +## Deploy + +Prior to using Pulumi, you need to login. There are some options here, which you can read about, but the simplest thing is to just use `--local`: + +``` +pulumi login --local +``` + +Now we can bring up the deployment. Create a stack when prompted with your name of choice. + +``` +pulumi up +``` + +If you want to destroy the deployment, bring it down like this: + +``` +pulumi down +``` + +## Replacing backends + +If you want to replace any data or metadata backends, just edit `vars.py` and run `pulumi up` again. Note that this is a destructive operation and any backends not present in the new config will be decomissioned. Data loss is possible if too many backends are decommissioned at one time without rebuilding the data. You must have the minimal shard count available to be able to reconstruct the data. + +After running `pulumi up` with the new config, the Pulumi script will automatically upload an updated Zstor config file to the VM. However, Zstor will not start using the new config automatically. You either need to restart Zstor or perform a hot reload of the config by sending the SIGUSR1 signal to Zstor: + +``` +pkill zstor -SIGUSR1 +``` + +Once the new config is loaded, Zstor will automatically start writing data or metadata to the new backends to restore the desired shard count for each stored file. This can take up to ten minutes to be triggered. + +You can check the progress of rebuilding using the Zstor `status` command: + +``` +zstor -c /etc/zstor-default.toml status +``` + +## Recover to new VM + +If you need to replace the frontend VM for any reason, such as a node outage, follow these steps. Any data that has been uploaded to the backends can be recovered into the new VM. Any data that was not yet uploaded to the backends will be lost. + +1. Update the `vars.py` file and set `VM_NODE` to the new node id +2. Destroy the old VM and deploy the new VM by running `pulumi up` +3. SSH to the new VM and run the recovery script: + +``` +bash /root/scripts/recover.sh +``` + +If all went well, your files should appear under the mount point, `/mnt/qsfs`. diff --git a/docs/06_operating.md b/docs/06_operating.md new file mode 100644 index 0000000..08248de --- /dev/null +++ b/docs/06_operating.md @@ -0,0 +1,110 @@ +## Introduction + +While some operational details are covered in the deployment sections above, this section provides an overview of operational concerns for the Quantum Safe Storage system. + +## Data rotation + +Data is uploaded from the frontend machine to the backends in two cases: + +1. A zdbfs data block is filled +2. The rotation timeout is reached (specified as zdbfs cli flag) + +This means that the rotation timeout, plus time spent processing and uploading data, represents an upper limit on how long data sent to the frontend might be lost if the frontend goes down. + +While a shorter rotation timeout means less potential for data loss, a longer timeout can mean that more data blocks get completely filled which is better for zdb performance. A timeout of 15 minutes is probably a good compromise for most use cases. + +## Monitoring + +Zstor exposes various metrics on a Prometheus endpoint, including metrics about the backends, zstor operationns, and also about the zdbfs process. + +Metrics are served at `localhost:9100/metrics`. + +These are the available metrics of each type: + +### Backend + + - Status of the connection to the 0-db + - Number of entries stored + - Size of data in bytes + - Data limit in bytes + - Size of index in bytes + - Number of I/O errors in index + - Number of faults in index + - Number of I/O errors in data + - Number of faults in data + - Free space in bytes on index disk + - Free space in bytes on data disk + +### Zstor + + - Store commands that finished + - Retrieve commands that finished + - Rebuild commands that finished + - Check commands that finished + +### Zdbfs + + - Total amount of fuse requests + - Total amount of cache hits in the filesystem + - Total amount of cache misses in the filesystem + - Total amount of times the cache was completely filled + - Total amount of linear flushes + - Total amount of random flushes + - Total amount of cache branches + - Amount of cache branches allocated + - Amount of memory cache entries + - Amount of blocks in the memory cache + - Amount of bytes used by cache blocks + - Total amount of syscalls done on the filesystem + - Total amount of bytes read from the filessytem + - Total amount of bytes written to the filessytem + - Total amount of errors returned by fuse calls + +## Visualize monitoring data with Grafana + +If you connect a Grafana instance to the Prometheus instance hosting metrics from zstor, you can import [this dashboard](https://scottyeager.grafana.net/public-dashboards/b522da8a37864e86bcc384ebdc5ae74e) to visualize the data. Note that the current dashboard is a first version and is not necessarily complete or optimal. + +## Maintenance + +If a backend goes offline, it can be replaced with a new one. Here are the steps to do this manually: + +1. Deploy new zdb with same or greater storage capacity +2. Update zstor config to use new zdb +3. Hot reload zstor config by issuing a `SIGUSR1` signal with `kill -SIGUSR1` (restarting zstor also works) +4. Zstor repair subsystem will automatically regenerate the data shards and store them in the new backend zdb. + +## Replace frontend + +If the frontend machine goes down, it's possible to recover the data into a new machine. Of course, only data that has actually been uploaded to the backends can be recovered (see note above about rotation timeout). + +The first step is to deploy and prepare a new machine. That means installing all necessary software components and uploading the zstor config file. Don't start up services on the new machine yet though. + +From there, the recovery steps involve priming the new machine with enough data from the backends that it can resume operations and reconstruct all stored data. These steps are demonstrated in a [script](https://github.com/threefoldtech/quantum-storage/blob/master/pulumi/scripts/recover.sh) provided with the Pulumi based deployer. + +## Zstor + +There are a few commands available for querying info from zstor on the cli. + +### Status + +This shows an overview of the configured backends and their status: + +``` +zstor -c /path/to/config status +``` + +### Check + +The `check` command simply checks that a given path has an entry in the metadata store. It does not actually check the integrity of the data. + +``` +zstor -c /path/to/config check /path/to/file +``` + +The file's checksum is printed, along with the path, if the metadata exists. + +Here's a command to generate the same checksum independently: + +``` +b2sum -l 128 /path/to/file +``` diff --git a/docs/troubleshooting.md b/docs/07_troubleshooting.md similarity index 89% rename from docs/troubleshooting.md rename to docs/07_troubleshooting.md index 52b212a..9756326 100644 --- a/docs/troubleshooting.md +++ b/docs/07_troubleshooting.md @@ -1,4 +1,6 @@ -# Troubleshooting +## Introduction + +Here's a collection of errors that might arise when using the Quantum Safe Storage system, along with some explanations and tips to resolve them. ## zstor status @@ -21,7 +23,7 @@ ERROR error during storage: ZDB at [300:cc55:8958:c1ff:229e:63d2:6fda:878d]:9900 Check that the zdb ip adress is reachable by performing a normal `ping` command. If it is, try a [redis-cli](https://redis.io/docs/ui/cli/) `PING` to check if the zdb is alive. -## fuse device not found +## Fuse device not found ```log fusermount3: fuse device not found, try 'modprobe fuse' first @@ -29,7 +31,7 @@ fusermount3: fuse device not found, try 'modprobe fuse' first Make sure fuse3 is installed and the kernel module is loaded. -## failed to process store command error during accessing local storage for attempting to store file which is not in the file tree rooted at +## Failed to process store command error during accessing local storage for attempting to store file which is not in the file tree rooted at ```log ERROR failed to process store command error during accessing local storage for attempting to store file which is not in the file tree rooted at /mnt: invalid data, queueing a retry @@ -37,7 +39,7 @@ ERROR failed to process store command error during accessing local storage for a The zstor config has an invalid `root` property set. Set it to where the local 0-db stores its data or drop it completely. -## could not find any viable backend distribution to statisfy redundancy requirement +## Could not find any viable backend distribution to statisfy redundancy requirement ```log 2023-03-03 13:25:56 +00:00: DEBUG Finding backend config diff --git a/docs/deployment.md b/docs/archive/diagram.md similarity index 100% rename from docs/deployment.md rename to docs/archive/diagram.md diff --git a/docs/encryption.md b/docs/encryption.md deleted file mode 100644 index ea56010..0000000 --- a/docs/encryption.md +++ /dev/null @@ -1,10 +0,0 @@ -# Encryption - -## encryption key - -The AES encryption keys are 32 random bytes in hexadecimal representation. -They can easily be generated on the commandline using openssl: - -```sh -openssl rand -hex 32 -``` diff --git a/docs/example_zstor_config.toml b/docs/example_zstor_config.toml deleted file mode 100644 index 6306e2d..0000000 --- a/docs/example_zstor_config.toml +++ /dev/null @@ -1,107 +0,0 @@ -minimal_shards = 8 -expected_shards = 12 -redundant_groups = 0 -redundant_nodes = 1 -zdbfs_mountpoint = "/mnt/qsfs" -socket = "/var/run/zstor.sock" -prometheus_port = 9100 -zdb_data_dir_path = "/data/data/zdbfs-data" -max_zdb_data_dir_size = 25600 - -[encryption] -algorithm = "AES" -key = "000001200000000001000300000004000a000f00b00000000000000000000000" - -[compression] -algorithm = "snappy" - -[meta] -type = "zdb" - -[meta.config] -prefix = "meta" - -[meta.config.encryption] -algorithm = "AES" -key = "0101010101010101010101010101010101010101010101010101010101010101" - -[[meta.config.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]:9900" -namespace = "741-18336-meta1" -password = "supersecretpass" - -[[meta.config.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-meta2" -password = "supersecretpass" - -[[meta.config.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-meta3" -password = "supersecretpass" - -[[meta.config.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-meta4" -password = "supersecretpass" - -[[groups]] -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data1" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data2" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data3" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data4" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data5" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data6" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data7" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data8" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data9" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data10" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data11" -password = "supersecretpass" - -[[groups.backends]] -address = "[300:db8:3333:4444:5555:6666:7777:8888]9900" -namespace = "741-18336-data12" -password = "supersecretpass" \ No newline at end of file diff --git a/docs/manual_setup.md b/docs/manual_setup.md deleted file mode 100644 index 2839325..0000000 --- a/docs/manual_setup.md +++ /dev/null @@ -1,70 +0,0 @@ -# Manual setup of QSFS - -This document explains how to manually set up QSFS on ubuntu. When using another linux distribution some steps can be slightly different. - -## prerequisistes - -### fuse - -Install the fuse kernel module : `apt-get update && apt-get install fuse3` - -### 0-db's - -In order to store the data in remote locations, you need to have 0-db's. -4 0-db's are needed for the metadata and m+n for the data, n being the number of 0-db's that can be lost without data loss ( m 0-db's are required to restore the data ). - -How to deploy 0-db's for qsfs on the Threefold grid is explained in the [manual](https://www2.manual.grid.tf/javascript/grid3_javascript_qsfs_zdbs.html). - -If the 0-db's are deployed on the grid, make sure you can connect to the Theefold Planetary network. -This is already working if you are setting up qsfs on a VM on the grid, if not this needs to be configured. - -## Directories - -A qsfs mount point is required and a directory for qsfs to store the temporary data. -This guide assumes `/mnt/qsfs` for the mount point and `/data` for the qsfs temporary data. Create them if they do not exist yet. - -## Install the individual components - -`wget` the latest released binaries from the following components: - -- 0-db-fs: : take the `amd64-linux-static` binary and save at `/bin/0-db-fs` -- 0-db: : take the static binary and sace at `/bin/0-db` -- 0-stor: : take `linux-musl` binary and save at `/bin/zstor` - -Make sure all binaries are executable:`chmod a+x /bin/0-db-fs /bin/0-db /bin/zstor` - -## 0-stor - -Adapt the [example zstor configuration](./example_zstor_config.toml) to use the previously created 0-db's, set an [encryption key](./encryption.md) and save it at `/etc/zstor_default/toml`. - -Now `zstor` can be started: `/usr/local/bin/zstor -c /etc/zstor_default.toml monitor`. If you don't want the process to block your terminal, you can start it in the background: `nohup /tmp/zstor -c /etc/zstor_default.toml monitor &`. - -## Local 0-db - -First we will get the [hook script](../lib/zdb-hook.sh). Download it to `/bin/zdb-hook.sh` and make sure it is executable (`chmod +x /bin/zdb-hook.sh`). - -The local 0-db which is used by 0-db-fs can now be started: - -```sh -/bin/0-db \ - --index /data/index \ - --data /data/data \ - --datasize 67108864 \ - --mode seq \ - --hook /bin/zdb-hook.sh \ - --background -``` - -## 0-db-fs - -Finally, we will start 0-db-fs. This guides opts to mount the fuse filesystem in `/mnt/qsfs`. - -```sh -/bin/0-db-fs /mnt/qsfs -o autons -o background -``` - -You should now have the qsfs filesystem mounted at `/mnt/qsfs`. As you write data, it will save it in the local 0-db, and it's data containers will be periodically encoded and uploaded to the backend data storage 0-db's. - -## It does not work - -Check the [troubleshooting guide](./troubleshooting.md). diff --git a/docs/operations.md b/docs/operations.md deleted file mode 100644 index cc05840..0000000 --- a/docs/operations.md +++ /dev/null @@ -1,23 +0,0 @@ -# QSFS Operation - -## zstor status - -## commandline - -the `zstor -c status` command gives an overview of the backend 0-db's, if they are reachable or not and the storage space they consume. - -## Monitoring, alerting and statistics - -0-stor collects metrics about the system. It can be configured with a 0-db-fs mountpoint, which will trigger 0-stor to collect 0-db-fs statistics, next to some 0-db statistics which are always collected. If the prometheus_port config option is set, 0-stor will serve metrics on this port for scraping by prometheus. - -They are available over http on the configured port at the `/metrics` path. -Test it with `curl localhost:9100/metrics` for example. - -## Some 0-db backends are broken - -If a 0-db backend is broken (the host is down for example) it needs to be replaced. - -Replace the malfunctioning 0-db's with new ones in the zstor config. -For zstor to reload the config, send it a `SIGUSR1` signal (`kill -SIGUSR1`) or restart the entire qsfs system. - -The 0-stor repair subsystem will take care of rebulding the data, regenerating the shards, and storing the new shards on the new 0-db's.