Fortis Storage Refactor

azuredeploy.json

@@ -111,7 +111,6 @@
         "deploymentVmSize": "Standard_D2_v2",
         "deploymentUbuntuOSVersion": "16.04.0-LTS",
         "imagePublisher": "Canonical",
-        "deisStorageAcctName": "k8deisstorage",
         "imageOffer": "UbuntuServer",
         "vmStorageAccountContainerName": "vhds",
         "nicName": "fortisVMNic",
@@ -165,17 +164,6 @@
             "name": "[variables('storageAccountName')]",
             "apiVersion": "2016-01-01",
             "location": "[resourceGroup().location]",
-            "sku": {
-                "name": "[variables('premStorageSku')]"
-            },
-            "kind": "Storage",
-            "properties": {}
-        },
-        {
-            "type": "Microsoft.Storage/storageAccounts",
-            "name": "[variables('deisStorageAcctName')]",
-            "apiVersion": "2016-01-01",
-            "location": "[resourceGroup().location]",
             "sku": {
                 "name": "[variables('vmStorageAccountType')]"
             },
@@ -510,7 +498,7 @@
         },
         "fortisAdminSiteURL": {
             "type": "string",
-            "value": "[reference(resourceId('Microsoft.Storage/storageAccounts', variables('deisStorageAcctName'))).tags.FORTIS_ADMIN_INTERFACE_HOST]"
+            "value": "[reference(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))).tags.FORTIS_ADMIN_INTERFACE_HOST]"
         },
         "agentFQDN": {
             "type": "string",

fortis-deploy.sh

@@ -13,7 +13,7 @@ Arguments
   --user_name|-un                    [Required] : Admin user name for the Kubernetes cluster
   --resource_group|-rg               [Required] : Resource group containing your Kubernetes cluster
   --master_fqdn|-mf                  [Required] : Master FQDN of your Kubernetes cluster
-  --storage_account_name|-san        [Required] : Premium Storage Account name used for Kubernetes's persistent storage
+  --storage_account_name|-san        [Required] : Storage Account name used for Kubernetes's persistent storage
   --storage_account_key|-sak         [Required] : Storage Account key used for Kubernetes persistent storage
   --spark_worker_count|-sw           [Required] : Spark Worker Node Count
   --cassandra_node_count|-cn         [Required] : Port used for Front50, defaulted to 8080
@@ -240,11 +240,13 @@ if ! (command -v helm >/dev/null); then
   install_helm
 fi
 
-echo "Installed Helm. Adding premium storage accounts."
+echo "Installed Helm. Adding storage share for spark checkpointing."
 
-#Create the K8 vhds storage container
-echo "creating vhds container"
-sudo az storage container create --name vhds --account-key="${storage_account_key}" --account-name="${storage_account_name}"
+#Create the K8 azure file storage container
+echo "creating azure file share"
+readonly checkpointfileshare="checkpoint"
+
+sudo az storage share create --name ${checkpointfileshare} --account-key="${storage_account_key}" --account-name="${storage_account_name}"
 
 sleep 10
 
@@ -269,4 +271,6 @@ chmod 752 create-cluster.sh
     "${site_name}" \
     "${eh_conn_str}" \
     "${sb_conn_str}" \
-    "${site_type}"
+    "${storage_account_key}" \
+    "${checkpointfileshare}" \
+    "${site_type}"

ops/create-cluster.sh

@@ -9,13 +9,12 @@ readonly app_insights_id="$6"
 readonly site_name="$7"
 readonly eh_conn_str="$8"
 readonly sb_conn_str="$9"
-readonly site_type="${10}"
+readonly storage_account_key="${10}"
+readonly checkpointfileshare="${11}"
+readonly site_type="${12}"
 
 chmod -R 752 .
 
-./create-disk.sh "${k8location}" "${storage_account_name}"
-sleep 10
-
 git clone https://github.com/CatalystCode/charts.git
 
 echo "Installing Cassandra chart"
@@ -24,7 +23,7 @@ echo "Finished. Now installing feature service DB"
 # shellcheck disable=SC1091
 . ./install-feature-service-db.sh "${k8location}" "${k8resource_group}"
 echo "Finished. Now installing DEIS"
-./install-deis.sh "${k8location}" "${k8resource_group}"
+./install-deis.sh "${k8location}" "${k8resource_group}" "${storage_account_name}" "${storage_account_key}"
 echo "Finished. Now installing DEIS fortis graphql service"
 ./deis-apps/fortis-services/create-app.sh
 echo "Finished. Now deploying"
@@ -79,10 +78,11 @@ echo "Finished. Deploying environment settings to cluster."
     "${eh_conn_str}" \
     "${feature_service_db_conn_str}" \
     "${fortis_central_directory}" \
-    "${sb_conn_str}"
+    "${sb_conn_str}" \
+    "${storage_account_name}"
 
 echo "Finished. Installing spark cluster."
-./install-spark.sh "${k8spark_worker_count}" "${spark_config_map_name}" "${fortis_central_directory}"
+./install-spark.sh "${k8spark_worker_count}" "${spark_config_map_name}" "${fortis_central_directory}" "${storage_account_name}" "${storage_account_key}" "${checkpointfileshare}"
 
 #./install-elasticsearch
 #./install-kibana

ops/install-deis.sh

@@ -2,22 +2,21 @@
 
 readonly k8location="$1"
 readonly k8resource_group="$2"
-readonly DEIS_STORAGE_ACCOUNT_NAME=k8deisstorage
+readonly storage_account_name="$3"
+readonly storage_account_key="$4"
 
 curl -sSL http://deis.io/deis-cli/install-v2.sh | bash
 sudo ln -fs "${PWD}/deis" /usr/local/bin/deis
 
 echo "creating deis storage account ${k8location}"
 id
 
-DEIS_STORAGE_ACCOUNT_KEY="$(az storage account keys list -n "${DEIS_STORAGE_ACCOUNT_NAME}" -g "${k8resource_group}" --query [0].value --output tsv)"
-
 echo "starting deis installation using helm"
 helm repo add deis https://charts.deis.com/workflow
 
 echo "Installing Deis on Cluster"
 
-helm install deis/workflow --name deis --namespace=deis --set global.storage=azure,azure.accountname="${DEIS_STORAGE_ACCOUNT_NAME}",azure.accountkey="${DEIS_STORAGE_ACCOUNT_KEY}",azure.registry_container=registry,azure.database_container=database,azure.builder_container=builder
+helm install deis/workflow --name deis --namespace=deis --set global.storage=azure,azure.accountname="${storage_account_name}",azure.accountkey="${storage_account_key}",azure.registry_container=registry,azure.database_container=database,azure.builder_container=builder
 
 echo "Looking up DEIS_ROUTER_HOST_ROOT"
 sleep 250

ops/install-spark.sh

@@ -5,11 +5,24 @@ if ! (command -v jq >/dev/null); then sudo apt install jq; fi
 readonly k8spark_worker_count="$1"
 readonly ConfigMapName="$2"
 readonly fortis_central_directory="$3"
+readonly storage_acct_name="$4"
+readonly storage_acct_key="$5"
+readonly checkpointfileshare="$6"
 readonly gh_fortis_spark_repo_name="project-fortis-spark"
 readonly latest_version=$(curl "https://api.github.com/repos/catalystcode/${gh_fortis_spark_repo_name}/releases/latest" | jq -r '.tag_name')
 readonly fortis_jar="fortis-${latest_version}.jar"
+
 readonly SparkCommand="spark-submit --deploy-mode cluster --driver-memory 4g --supervise --master spark://spark-master:7077 --verbose --class com.microsoft.partnercatalyst.fortis.spark.ProjectFortis \"https://fortiscentral.blob.core.windows.net/jars/${fortis_jar}\""
 
 cd charts || exit -2
-helm install --set Worker.Replicas="${k8spark_worker_count}"  --set Worker.ConfigMapName="${ConfigMapName}" --set Master.ConfigMapName="${ConfigMapName}" --set Master.SparkSubmitCommand="${SparkCommand}" --name spark-cluster ./stable/spark --namespace spark
-cd ..
+helm install --set Worker.Replicas="${k8spark_worker_count}" \
+             --set Master.ImageTag="2.2.0" \
+             --set Worker.ImageTag="2.2.0" \
+             --set Worker.ConfigMapName="${ConfigMapName}" \
+             --set Master.ConfigMapName="${ConfigMapName}" \
+             --set Master.SparkSubmitCommand="${SparkCommand}" \
+             --set Persistence.PvcAcctName="${storage_acct_name}" \
+             --set Persistence.PvcPwd="${storage_acct_key}" \
+             --set Persistence.CheckpointShare="${checkpointfileshare}" \
+             --name spark-cluster ./stable/spark --namespace spark
+cd ..

ops/setup-environment.sh

@@ -15,6 +15,7 @@ readonly eh_conn_str="${9}"
 readonly feature_service_db_conn_str="${10}"
 readonly fortis_central_directory="${11}"
 readonly sb_conn_str="${12}"
+readonly storage_account_name="${13}"
 
 readonly fortis_admin_interface="http://${fortis_interface_host}/#/site/${site_name}/admin"
 readonly default_language="en"
@@ -67,4 +68,4 @@ deis config:set DEFAULT_SITE_NAME="${site_name}"
 cd ../../ || exit 2
 
 #Set the deployed service host url tag so we can output that on the deployment console to the user
-sudo az resource tag --tags FORTIS_INTERFACE_HOST="${fortis_interface_host}" FORTIS_ADMIN_INTERFACE_HOST="${fortis_admin_interface}" FORTIS_SERVICE_HOST="${graphql_service_host}" -g "${k8resource_group}" -n k8deisstorage --resource-type "Microsoft.Storage/storageAccounts"
+sudo az resource tag --tags FORTIS_INTERFACE_HOST="${fortis_interface_host}" FORTIS_ADMIN_INTERFACE_HOST="${fortis_admin_interface}" FORTIS_SERVICE_HOST="${graphql_service_host}" -g "${k8resource_group}" -n "${storage_account_name}" --resource-type "Microsoft.Storage/storageAccounts"