Merge pull request #953 from supertetelman/common-scripts

Introduce a common script library, config for env vars, and inject these into all scripts

config.example/env.sh

@@ -0,0 +1,6 @@
+# This file acts as a location to override the default configurations of deepops/scripts/*
+# Many of the scripts in this directory define global variables and set reasonable defaults
+# Global variables (in all caps) that are defined here will be automatically sourced and used in all scripts
+# See deepops/scripts/common.sh for implementation details
+
+DEEPOPS_EXAMPLE_VAR=""

docs/deepops/configuration.md

@@ -12,6 +12,7 @@ In particular, this directory includes:
 - `config/group_vars/all.yml`: An Ansible [variables file](https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html) that contains variables we expect to work for all hosts
 - `config/group_vars/k8s-cluster.yml`: Variables specific to deploying Kubernetes clusters
 - `config/group_vars/slurm-cluster.yml`: Variables specific to deploying Slurm clusters
+- `config/env.sh`: Global variables that override default variable values for all `sh` files in `scripts/*`.
 - `config/requirements.yml`: An Ansible Galaxy [requirements file](https://docs.ansible.com/ansible/latest/galaxy/user_guide.html#installing-roles-and-collections-from-the-same-requirements-yml-file) that contains a list of custom Collections and Roles to install. Collections and Roles required by DeepOps are stored in a separate `roles/requirements.yml` file, which should not be modified.
 
 It's expected that most DeepOps deployments will make changes to these files!

scripts/airgap/build_offline_cache.sh

@@ -4,6 +4,10 @@ set -ex
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
 DEEPOPS_CONFIG_DIR="${DEEPOPS_CONFIG_DIR:-${ROOT_DIR}/config.example}"
+
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
 DEST_DIR="/tmp/deepops"
 TARBALL="/tmp/deepops-archive.tar"
 DEEPOPS_BUILD_TARBALL="${DEEPOPS_BUILD_TARBALL:-1}"

scripts/common.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# This is a common set of libraries, configuration override, helper functions, and debug output
+# This file should be sourced at the top of all scripts and primarily does 3 things
+#  1. Will source the env.sh file to allow override variables be version controlled in ./config
+#  2. Will print out some standard debug for each script, to ease debugging
+#  3. Will provide a common set of libraries, directory names, etc.
+
+
+# Determine the path to the configuration directory and verify it exists
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/.."
+DEEPOPS_CONFIG_DIR=${DEEPOPS_CONFIG_DIR:-"${ROOT_DIR}/config"}
+if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then
+  # Because this is a widely used script, we warn here instead of throwing an error
+  echo "WARNING: Can't find configuration in ${DEEPOPS_CONFIG_DIR}"
+  echo "WARNING: Please set DEEPOPS_CONFIG_DIR env variable to point to config location"
+else
+  # Source the configuration environment variable overrides
+  source ${DEEPOPS_CONFIG_DIR}/env.sh
+fi
+
+# Print out base debug
+echo "Starting '${0}';  DeepOps version '${DEEPOPS_VERSION}'"

scripts/generic/install_docker.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 DOCKER_COMPOSE_URL="${DOCKER_COMPOSE_URL:-https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)}"
 
 type docker >/dev/null 2>&1

scripts/k8s/debug.sh

@@ -7,6 +7,11 @@
 # Requirements for this script are a working "kubectl" and ideally a working "helm"
 # Optionally, a working "ansible" with a config/inventory file that has kubernetes node defined in a kube-node group
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 timestamp=$(date +%s)
 logdir=config/log_${timestamp}
 mkdir -p ${logdir}
@@ -35,6 +40,8 @@ kubectl get nodes > ${logdir}/get-nodes.log
 kubectl describe nodes > ${logdir}/describe-nodes.log
 kubectl get storageclass > ${logdir}/get-storageclass.log
 kubectl get events -A > ${logdir}/get-events.log
+kubectl get svc -A > ${logdir}/get-svc.log
+
 # Kubectl / GPU Operator (Generic for any Kubernetes cluster)
 kubectl get pvc -A > ${logdir}/get-pvc.log
 for pod in $(kubectl get pods -n gpu-operator-resources  | grep nvidia-device-plugin | awk '{print $1}'); do

scripts/k8s/deploy_dashboard_user.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 # Make the dashboard a NodePort
 kubectl patch svc -n kube-system kubernetes-dashboard  -p '{"spec": {"type": "NodePort", "ports": [{"nodePort": 31443, "port": 443}] }}'
 

scripts/k8s/deploy_ingress.sh

@@ -5,6 +5,9 @@ set -x
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
 
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
 HELM_CHARTS_REPO_INGRESS="${HELM_CHARTS_REPO_INGRESS:-https://kubernetes.github.io/ingress-nginx}"
 HELM_INGRESS_CHART_VERSION="${HELM_INGRESS_CHART_VERSION:-3.5.1}"
 # HELM_INGRESS_CONFIG, defaults below based on presence of metallb

scripts/k8s/deploy_kubeflow.sh

@@ -5,6 +5,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
 CONFIG_DIR="${ROOT_DIR}/config"
 
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
 # Specify credentials for the default user.
 # TODO: Dynamically sed/hash these value into the CONFIG, these are currently not used
 export KUBEFLOW_USER_EMAIL="${KUBEFLOW_USER_EMAIL:-admin@kubeflow.org}"

scripts/k8s/deploy_loadbalancer.sh

@@ -5,6 +5,9 @@ set -x
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
 
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
 # Allow overriding config dir to look in
 DEEPOPS_CONFIG_DIR=${DEEPOPS_CONFIG_DIR:-"${ROOT_DIR}/config"}
 if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then

scripts/k8s/deploy_monitoring.sh

@@ -11,6 +11,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
 cd "${ROOT_DIR}" || exit 1
 
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
 # Allow overriding config dir to look in
 DEEPOPS_CONFIG_DIR=${DEEPOPS_CONFIG_DIR:-"${ROOT_DIR}/config"}
 

scripts/k8s/deploy_rook.sh

@@ -8,8 +8,11 @@
 # Get absolute path for script and root
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="${SCRIPT_DIR}/../.."
-CHART_VERSION="1.22.1"
 
+# Source common libraries and env variables
+source ${ROOT_DIR}/scripts/common.sh
+
+CHART_VERSION="1.22.1"
 HELM_ROOK_CHART_REPO="${HELM_ROOK_CHART_REPO:-https://charts.rook.io/release}"
 HELM_ROOK_CHART_VERSION="${HELM_ROOK_CHART_VERSION:-v1.1.1}"
 

scripts/k8s/install_helm.sh

@@ -2,6 +2,11 @@
 
 set -x
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 HELM_INSTALL_DIR=/usr/local/bin
 HELM_INSTALL_SCRIPT_URL="${HELM_INSTALL_SCRIPT_URL:-https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3}"
 HELM_MINIMUM_VERSION=v3.4.1+gc4e7485

scripts/k8s/setup_remote_k8s.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 KUBECTL_BINARY_URL="${KUBECTL_BINARY_URL:-https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl}"
 
 # Install dependencies

scripts/k8s/verify_gpu.sh

@@ -4,6 +4,11 @@
 # Check the output and verify the number of nodes and GPUs is as expected
 # TODO: This script should be wrapped by Ansible to verify that the output of nvidia-smi on each node matches K8S
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 export KFCTL=${KFCTL:-~/kfctl}
 export CLUSTER_VERIFY_NS=${CLUSTER_VERIFY_NS:-cluster-gpu-verify}
 export CLUSTER_VERIFY_EXPECTED_PODS=${CLUSTER_VERIFY_EXPECTED_PODS:-}

scripts/nginx-docker-cache/gen-ca.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+# Source common libraries and env variables
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="${SCRIPT_DIR}/../.."
+source ${ROOT_DIR}/scripts/common.sh
+
 CA_CRT_OUTFILE="${CA_CRT_OUTFILE:-/tmp/ca.crt}"
 CA_KEY_OUTFILE="${CA_KEY_OUTFILE:-/tmp/ca.key}"