From 85035ba76093d19af505308154f1c4033f69b40c Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Tue, 12 Apr 2022 11:38:51 -0400
Subject: [PATCH] democratic-csi plugin

---
 packs/democratic_csi_nfs/CHANGELOG.md         |   3 +
 packs/democratic_csi_nfs/README.md            | 170 ++++++++++++++++++
 packs/democratic_csi_nfs/metadata.hcl         |  11 ++
 packs/democratic_csi_nfs/outputs.tpl          |  24 +++
 .../templates/_constraints.tpl                |  24 +++
 .../templates/_location.tpl                   |   5 +
 .../templates/_plugin_config_file.tpl         |  19 ++
 .../templates/_resources.tpl                  |   6 +
 .../templates/controller.nomad.tpl            |  56 ++++++
 .../templates/node.nomad.tpl                  |  51 ++++++
 packs/democratic_csi_nfs/variables.hcl        | 120 +++++++++++++
 11 files changed, 489 insertions(+)
 create mode 100644 packs/democratic_csi_nfs/CHANGELOG.md
 create mode 100644 packs/democratic_csi_nfs/README.md
 create mode 100644 packs/democratic_csi_nfs/metadata.hcl
 create mode 100644 packs/democratic_csi_nfs/outputs.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/_constraints.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/_location.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/_plugin_config_file.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/_resources.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/controller.nomad.tpl
 create mode 100644 packs/democratic_csi_nfs/templates/node.nomad.tpl
 create mode 100644 packs/democratic_csi_nfs/variables.hcl

diff --git a/packs/democratic_csi_nfs/CHANGELOG.md b/packs/democratic_csi_nfs/CHANGELOG.md
new file mode 100644
index 0000000..3fafbcd
--- /dev/null
+++ b/packs/democratic_csi_nfs/CHANGELOG.md
@@ -0,0 +1,3 @@
+# 0.1.0
+
+- Initial release
diff --git a/packs/democratic_csi_nfs/README.md b/packs/democratic_csi_nfs/README.md
new file mode 100644
index 0000000..46130a1
--- /dev/null
+++ b/packs/democratic_csi_nfs/README.md
@@ -0,0 +1,170 @@
+# `democratic-csi` CSI plugin
+
+This pack deploys two jobs that run the
+[`democratic-csi`](https://github.com/democratic-csi/democratic-csi)
+CSI plugin. The node plugin tasks will be run as a system job, and the
+controller tasks will be run as a service job.
+
+## Client Requirements
+
+This pack can only be run on Nomad clients that have enabled volumes and
+privileged mode for the Docker task driver. In addition, clients will
+need to have a source NFS volume mounted to any client host that runs
+the controller task.
+
+### Example NFS Server
+
+The following is an example of installing and configuring a NFS server
+on an apt-based Linux distrobution (ex. Debian or Ubuntu). This
+configuration exports the directory `/var/nfs/general` to any Nomad
+client on the `192.168.56.0/24` address space (this is commonly used
+for Vagrant hosts on Virtualbox).
+
+```sh
+sudo apt-get install nfs-kernel-server
+sudo mkdir /var/nfs/general
+
+sudo cat <<EOF > /etc/exports
+/var/nfs/general 192.168.56.0/24(rw,sync,no_subtree_check,no_root_squash)
+EOF
+
+sudo systemctl enable nfs-kernel-server
+sudo systemctl start nfs-kernel-server
+```
+
+The `democratic-csi` controller is unusual in needing to mount the NFS
+volume because NFS doesn't have a remote API other than filesystem
+operations. So this plugin has to have the NFS export bind-mounted
+into the plugin container. Any client running the controller will need
+to have the NFS export in the host's `/etc/fstab`. The following
+configuration is an example for the NFS export shown above, assuming
+that the NFS server can be found at IP `192.168.56.60`. The mount
+point `/srv/nfs_data` shown here should be used for the
+`nfs_controller_mount_path` variable.
+
+```
+sudo cat <<EOF >> /etc/fstab
+192.168.56.60:/var/nfs/general /srv/nfs_data nfs4 rw,relatime 0 0
+EOF
+
+```
+
+## Variables
+
+The following variables are required:
+
+* `nfs_share_host` - The IP address of the host for the NFS share.
+* `nfs_share_base_path` - The base directory exported from the NFS
+  share host.
+* `nfs_controller_mount_path` - The path where the NFS mount is
+  mounted as a host volume for the controller plugin.
+
+The following variables are optional:
+
+* `job_name` (string "democratic_csi") - The prefix to use as the job
+  name for the plugins. For exmaple, if `job_name = "democratic_csi"`,
+  the plugin job will be named `democratic_csi_controller`.
+* `datacenters` (list(string) ["dc1"]) - A list of datacenters in the
+  region which are eligible for task placement.
+* `region` (string "global") - The region where the job should be
+  placed.
+* `plugin_id` (string "org.democratic-csi.nfs") - The ID to register
+  in Nomad for the plugin.
+* `plugin_namespace` (string "default") - The namespace for the plugin
+  job.
+* `plugin_image` (string
+  "docker.io/democraticcsi/democratic-csi:latest") - The container
+  image for `democratic-csi`.
+* `plugin_csi_spec_version` (string "1.5.0") - The CSI spec version
+  that democratic-csi will comply with.
+* `plugin_log_level` (string "debug") - The log level for the plugin.
+* `nfs_dir_permissions_mode` (string "0777") - The unix file
+  permissions mode for the created volumes.
+* `nfs_dir_permissions_user` (string "root") - The unix user that owns
+  the created volumes.
+* `nfs_dir_permissions_group` (string "root") - The unix group that
+  owns the created volumes.
+* `controller_count` (number 2) - The number of controller instances
+  to be deployed (at least 2 recommended).
+* `volume_id` (string "myvolume") - ID for the example volume spec to
+  output.
+* `volume_namespace` (string "default") - Namespace for the example
+  volume spec to output.
+
+#### `constraints` List of Objects
+
+[Nomad job specification
+constraints](https://www.nomadproject.io/docs/job-specification/constraint)
+allow restricting the set of eligible nodes on which the tasks will
+run. This pack automatically configures the following required
+constraints:
+
+* Plugin tasks will run on Linux hosts only
+* Plugin tasks will run on hosts with the Docker driver's
+  [`volumes`](https://www.nomadproject.io/docs/drivers/docker#volumes-1)
+  enabled and
+  [`allow_privileged`](https://www.nomadproject.io/docs/drivers/docker#allow_privileged)
+  set to `true`.
+* The controller plugin tasks will be deployed on distinct hosts.
+
+You can set additional constraints with the `constraints` variable,
+which takes a list of objects with the following fields:
+
+* `attribute` (string) - Specifies the name or reference of the
+  attribute to examine for the constraint.
+* `operator` (string) - Specifies the comparison operator. The
+  ordering is compared lexically.
+* `value` (string) - Specifies the value to compare the attribute
+  against using the specified operation.
+
+Below is also an example of how to pass `constraints` to the CLI with
+with the `-var` argument.
+
+```bash
+nomad-pack run -var 'constraints=[{"attribute":"$${meta.my_custom_value}","operator":">","value":"3"}]' packs/democratic_csi_nfs
+```
+
+#### `resources` Object
+
+* `cpu` (number 500) - Specifies the CPU required to run this task in
+  MHz.
+* `memory` (number 256) - Specifies the memory required in MB.
+
+## Volume creation
+
+This pack outputs an example volume specification based on the plugin variables.
+
+#### **`volume.hcl`**
+
+```hcl
+type         = "csi"
+id           = "my_volume"
+namespace    = "default"
+name         = "my_volume"
+plugin_id    = "org.democratic-csi.nfs"
+
+capability {
+  access_mode     = "multi-node-multi-writer"
+  attachment_mode = "file-system"
+}
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+
+capability {
+  access_mode     = "single-node-reader-only"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  mount_flags = ["noatime"]
+}
+```
+
+Create this volume with the following command:
+
+```sh
+nomad volume create volume.hcl
+```
diff --git a/packs/democratic_csi_nfs/metadata.hcl b/packs/democratic_csi_nfs/metadata.hcl
new file mode 100644
index 0000000..8f96a5e
--- /dev/null
+++ b/packs/democratic_csi_nfs/metadata.hcl
@@ -0,0 +1,11 @@
+app {
+  url    = "https://github.com/democratic-csi/democratic-csi"
+  author = "Tim Gross <tgross@hashicorp.com>"
+}
+
+pack {
+  name        = "democratic_csi_nfs"
+  description = "This pack deploys the democratic-csi plugin, configured for use with NFS"
+  url         = "https://github.com/hashicorp/nomad-pack-community-registry/democratic_csi_nfs"
+  version     = "0.1.0"
+}
diff --git a/packs/democratic_csi_nfs/outputs.tpl b/packs/democratic_csi_nfs/outputs.tpl
new file mode 100644
index 0000000..bee7df4
--- /dev/null
+++ b/packs/democratic_csi_nfs/outputs.tpl
@@ -0,0 +1,24 @@
+type         = "csi"
+id           = "[[ .my.volume_id ]]"
+namespace    = "[[ .my.volume_namespace ]]"
+name         = "[[ .my.volume_id ]]"
+plugin_id    = "[[ .my.plugin_id ]]"
+
+capability {
+  access_mode     = "multi-node-multi-writer"
+  attachment_mode = "file-system"
+}
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+
+capability {
+  access_mode     = "single-node-reader-only"
+  attachment_mode = "file-system"
+}
+
+mount_options {
+  mount_flags = ["noatime"]
+}
diff --git a/packs/democratic_csi_nfs/templates/_constraints.tpl b/packs/democratic_csi_nfs/templates/_constraints.tpl
new file mode 100644
index 0000000..6811364
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/_constraints.tpl
@@ -0,0 +1,24 @@
+[[- define "constraints" -]]
+constraint {
+  attribute = "${attr.kernel.name}"
+  value     = "linux"
+}
+
+constraint {
+  attribute = "${attr.driver.docker.privileged.enabled}"
+  value     = true
+}
+
+[[ if .my.constraints ]][[ range $idx, $constraint := .my.constraints ]]
+constraint {
+  attribute = [[ $constraint.attribute | quote ]]
+  [[- if $constraint.value ]]
+  value     = [[ $constraint.value | quote ]]
+  [[- end ]]
+  [[- if $constraint.operator  ]]
+  operator  = [[ $constraint.operator | quote ]]
+  [[- end ]]
+}
+[[- end ]][[- end ]]
+
+[[- end -]]
diff --git a/packs/democratic_csi_nfs/templates/_location.tpl b/packs/democratic_csi_nfs/templates/_location.tpl
new file mode 100644
index 0000000..749cf57
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/_location.tpl
@@ -0,0 +1,5 @@
+[[- define "location" -]]
+namespace   = "[[ .my.plugin_namespace ]]"
+region      = "[[ .my.region ]]"
+datacenters = [[ .my.datacenters | toPrettyJson ]]
+[[- end -]]
diff --git a/packs/democratic_csi_nfs/templates/_plugin_config_file.tpl b/packs/democratic_csi_nfs/templates/_plugin_config_file.tpl
new file mode 100644
index 0000000..5d94464
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/_plugin_config_file.tpl
@@ -0,0 +1,19 @@
+[[- define "plugin_config_file" -]]
+
+template {
+  destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+  data = <<EOH
+driver: nfs-client
+instance_id:
+nfs:
+  shareHost: [[ .my.nfs_share_host ]]
+  shareBasePath: "[[ .my.nfs_share_base_path ]]"
+  controllerBasePath: "/storage"
+  dirPermissionsMode: "[[ .my.nfs_dir_permissions_mode ]]"
+  dirPermissionsUser: [[ .mynfs_dir_permissions_user ]]
+  dirPermissionsGroup: [[ .mynfs_dir_permissions_group ]]
+EOH
+}
+
+[[- end -]]
diff --git a/packs/democratic_csi_nfs/templates/_resources.tpl b/packs/democratic_csi_nfs/templates/_resources.tpl
new file mode 100644
index 0000000..f5cfcd3
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/_resources.tpl
@@ -0,0 +1,6 @@
+[[- define "resources" -]]
+resources {
+  cpu    = [[ .democratic_csi_nfs.resources.cpu ]]
+  memory = [[ .my.resources.memory ]]
+}
+[[- end -]]
diff --git a/packs/democratic_csi_nfs/templates/controller.nomad.tpl b/packs/democratic_csi_nfs/templates/controller.nomad.tpl
new file mode 100644
index 0000000..928eadd
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/controller.nomad.tpl
@@ -0,0 +1,56 @@
+job "[[ .my.job_name ]]_controller" {
+
+  [[ template "location" ]]
+
+  group "controller" {
+
+    count = [[ .my.controller_count ]]
+
+    [[ template "constraints" ]]
+
+    constraint {
+      operator = "distinct_hosts"
+      value    = "true"
+    }
+
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "[[ .my.plugin_image ]]"
+
+        args = [
+          "--csi-version=[[ .my.plugin_csi_spec_version ]]",
+          "--csi-name=[[ .my.plugin_id ]]",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=[[ .my.plugin_log_level ]]",
+          "--csi-mode=controller",
+          "--server-socket=${CSI_ENDPOINT}",
+        ]
+
+        # normally not required for controller plugins, but NFS
+        # doesn't have a remote API other than mounting, so this
+        # plugin has to be able to mount the NFS volume as a
+        # bind-mount in order to create and snapshot volumes.
+        privileged = true
+        mount {
+          type     = "bind"
+          source   = "[[ .my.nfs_controller_mount_path ]]"
+          target   = "/storage"
+          readonly = false
+        }
+      }
+
+      [[ template "plugin_config_file" ]]
+
+      [[ template "resources" ]]
+
+      csi_plugin {
+        id        = "[[ .my.plugin_id ]]"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+    }
+  }
+}
diff --git a/packs/democratic_csi_nfs/templates/node.nomad.tpl b/packs/democratic_csi_nfs/templates/node.nomad.tpl
new file mode 100644
index 0000000..ce21206
--- /dev/null
+++ b/packs/democratic_csi_nfs/templates/node.nomad.tpl
@@ -0,0 +1,51 @@
+job "[[ .my.job_name ]]_node" {
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  [[ template "location" ]]
+
+  group "node" {
+
+    [[ template "constraints" ]]
+
+    task "plugin" {
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+      }
+
+      config {
+        image = "[[ .my.plugin_image ]]"
+
+        args = [
+          "--csi-version=[[ .my.plugin_csi_spec_version ]]",
+          "--csi-name=[[ .my.plugin_id ]]",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=[[ .my.plugin_log_level ]]",
+          "--csi-mode=node",
+          "--server-socket=${CSI_ENDPOINT}",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged   = true
+        ipc_mode     = "host"
+        network_mode = "host"
+      }
+
+      [[ template "plugin_config_file" ]]
+
+      [[ template "resources" ]]
+
+      csi_plugin {
+        id        = "[[ .my.plugin_id ]]"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+    }
+  }
+}
diff --git a/packs/democratic_csi_nfs/variables.hcl b/packs/democratic_csi_nfs/variables.hcl
new file mode 100644
index 0000000..8448442
--- /dev/null
+++ b/packs/democratic_csi_nfs/variables.hcl
@@ -0,0 +1,120 @@
+variable "job_name" {
+  description = "The prefix to use as the job name for the plugins (ex. democratic_csi_controller for the controller)."
+  type        = string
+  default     = "democratic_csi"
+}
+
+variable "datacenters" {
+  description = "A list of datacenters in the region which are eligible for task placement."
+  type        = list(string)
+  default     = ["dc1"]
+}
+
+variable "region" {
+  description = "The region where the job should be placed."
+  type        = string
+  default     = "global"
+}
+
+variable "plugin_id" {
+  description = "The ID to register in Nomad for the plugin."
+  type        = string
+  default     = "org.democratic-csi.nfs"
+}
+
+variable "plugin_namespace" {
+  description = "The namespace for the plugin job."
+  type        = string
+  default     = "default"
+}
+
+variable "plugin_image" {
+  description = "The container image for democratic-csi."
+  type        = string
+  default     = "docker.io/democraticcsi/democratic-csi:latest"
+}
+
+variable "plugin_csi_spec_version" {
+  description = "The CSI spec version that democratic-csi will comply with."
+  type        = string
+  default     = "1.5.0"
+}
+
+variable "plugin_log_level" {
+  description = "The log level for the plugin."
+  type        = string
+  default     = "debug"
+}
+
+variable "nfs_share_host" {
+  description = "The IP address of the host for the NFS share."
+  type        = string
+}
+
+variable "nfs_share_base_path" {
+  description = "The base directory exported from the NFS share host."
+  type        = string
+}
+
+variable "nfs_controller_mount_path" {
+  description = "The path where the NFS mount is mounted as a host volume for the controller plugin."
+  type        = string
+}
+
+variable "nfs_dir_permissions_mode" {
+  description = "The unix file permissions mode for the created volumes"
+  type        = string
+  default     = "0777"
+}
+
+variable "nfs_dir_permissions_user" {
+  description = "The unix user that owns the created volumes."
+  type        = string
+  default     = "root"
+}
+
+variable "nfs_dir_permissions_group" {
+  description = "The unix group that owns the created volumes."
+  type        = string
+  default     = "root"
+}
+
+variable "controller_count" {
+  description = "The number of controller instances to be deployed (at least 2 recommended)."
+  type        = number
+  default     = 2
+}
+
+variable "constraints" {
+  description = "Additional constraints to apply to the jobs."
+  type        = list(object({
+    attribute = string
+    operator  = string
+    value     = string
+  }))
+  default = []
+}
+
+variable "resources" {
+  description = "The resources to assign to the plugin tasks."
+  type = object({
+    cpu    = number
+    memory = number
+  })
+  default = {
+    cpu    = 500,
+    memory = 256
+  }
+}
+
+variable "volume_id" {
+  description = "ID for the example volume spec to output."
+  type        = string
+  default     = "myvolume"
+}
+
+variable "volume_namespace" {
+  description = "Namespace for the example volume spec to output."
+  type        = string
+  default     = "default"
+}