-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: vsoch <vsoch@users.noreply.github.com>
- Loading branch information
Showing
1 changed file
with
215 additions
and
0 deletions.
There are no files selected for viewing
215 changes: 215 additions & 0 deletions
215
tests/lammps/data/k8s-size-4-local/.scripts/minicluster-submit-size-2.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,215 @@ | ||
#!/bin/bash | ||
|
||
# This is a template that will be populated with variables by Flux-Cloud | ||
# We only run it to check if a MiniCluster is running. An apply is only | ||
# needed if the MiniCluster is not created yet. | ||
|
||
# Include shared helper scripts | ||
# Colors | ||
red='\033[0;31m' | ||
green='\033[0;32m' | ||
yellow='\033[0;33m' | ||
blue='\033[0;34m' | ||
magenta='\033[0;35m' | ||
cyan='\033[0;36m' | ||
clear='\033[0m' | ||
|
||
function print_red() { | ||
echo -e "${red}$@${clear}" | ||
} | ||
function print_yellow() { | ||
echo -e "${yellow}$@${clear}" | ||
} | ||
function print_green() { | ||
echo -e "${green}$@${clear}" | ||
} | ||
function print_blue() { | ||
echo -e "${blue}$@${clear}" | ||
} | ||
function print_magenta() { | ||
echo -e "${magenta}$@${clear}" | ||
} | ||
function print_cyan() { | ||
echo -e "${cyan}$@${clear}" | ||
} | ||
|
||
function is_installed () { | ||
# Determine if a command is available for use! | ||
cmd="${1}" | ||
if command -v $cmd >/dev/null; then | ||
echo "$cmd is installed" | ||
else | ||
echo "$cmd could not be found" | ||
exit 1 | ||
fi | ||
} | ||
|
||
function install_operator() { | ||
# Shared function to install the operator from a specific repository branch and cleanup | ||
script_dir=${1} | ||
repository=${2} | ||
branch=${3} | ||
tmpfile="${script_dir}/flux-operator.yaml" | ||
run_echo wget -O $tmpfile https://raw.githubusercontent.com/${repository}/${branch}/examples/dist/flux-operator.yaml | ||
kubectl apply -f $tmpfile | ||
} | ||
|
||
|
||
function run_echo() { | ||
# Show the user the command then run it | ||
echo | ||
print_green "$@" | ||
retry $@ | ||
} | ||
|
||
function run_echo_allow_fail() { | ||
echo | ||
print_green "$@" | ||
$@ || true | ||
} | ||
|
||
function retry() { | ||
# Retry an unsuccessful user command, per request | ||
while true | ||
do | ||
$@ | ||
retval=$? | ||
if [[ "${retval}" == "0" ]]; then | ||
return | ||
fi | ||
print_blue "That command was not successful. Do you want to try again? 🤔️" | ||
read -p " (yes/no) " answer | ||
# Exit with non-zero response so we know to stop in script. | ||
case ${answer} in | ||
yes ) continue;; | ||
no ) echo exiting...; | ||
exit 1;; | ||
* ) echo invalid response; | ||
exit 1;; | ||
esac | ||
done | ||
} | ||
|
||
|
||
function prompt() { | ||
# Prompt the user with a yes/no command to continue or exit | ||
print_blue "$@ 🤔️" | ||
read -p " (yes/no) " answer | ||
case ${answer} in | ||
yes ) echo ok, we will proceed;; | ||
no ) echo exiting...; | ||
exit 1;; | ||
* ) echo invalid response; | ||
exit 1;; | ||
esac | ||
} | ||
|
||
|
||
function with_exponential_backoff { | ||
# Run with exponential backoff - assume containers take a while to pull | ||
local max_attempts=100 | ||
local timeout=1 | ||
local attempt=0 | ||
local exitcode=0 | ||
|
||
while [[ $attempt < $max_attempts ]]; do | ||
"$@" | ||
exitcode=$? | ||
|
||
if [[ $exitcode == 0 ]]; then | ||
break | ||
fi | ||
|
||
echo "Failure! Retrying in $timeout.." 1>&2 | ||
sleep $timeout | ||
attempt=$(( attempt + 1 )) | ||
timeout=$(( timeout * 2 )) | ||
done | ||
|
||
if [[ $exitCode != 0 ]]; then | ||
echo "You've failed me for the last time! ($@)" 1>&2 | ||
fi | ||
return $exitcode | ||
} | ||
|
||
NAMESPACE="flux-operator" | ||
CRD="/home/vanessa/Desktop/Code/flux/flux-cloud/tests/lammps/data/k8s-size-4-local/.scripts/minicluster.yaml" | ||
JOB="lammps" | ||
|
||
print_magenta " apply : ${CRD}" | ||
print_magenta " job : ${JOB}" | ||
|
||
is_installed kubectl | ||
|
||
# Create the namespace (ok if already exists) | ||
run_echo_allow_fail kubectl create namespace ${NAMESPACE} | ||
|
||
# Ensure we have a MiniCluster of the right namespace running | ||
echo | ||
podsCleaned="false" | ||
brokerPrefix="${JOB}-0" | ||
print_blue "Looking for existing MiniCluster..." | ||
state=$(kubectl get pods --namespace ${NAMESPACE} 2>&1) | ||
lines=$(echo $state | wc -l) | ||
if [[ "${lines}" == "1" ]] && [[ "${state}" == *"No resources found in"* ]]; then | ||
echo | ||
print_green "🌀️ Creating MiniCluster in ${NAMESPACE}" | ||
# Apply the job, get pods | ||
run_echo kubectl apply -f ${CRD} | ||
run_echo kubectl get -n ${NAMESPACE} pods | ||
|
||
# continue until we find the index-0 pod | ||
brokerPrefix="${JOB}-0" | ||
brokerReady="false" | ||
|
||
echo | ||
print_blue "Waiting for broker pod with prefix ${brokerPrefix} to be created..." | ||
while [[ "${brokerReady}" == "false" ]]; do | ||
echo -n "." | ||
sleep 2 | ||
for pod in $(kubectl get pods --selector=job-name=${JOB} --namespace ${NAMESPACE} --output=jsonpath='{.items[*].metadata.name}'); do | ||
if [[ "${pod}" == ${brokerPrefix}* ]]; then | ||
echo | ||
print_green "🌀️ Broker pod is created." | ||
brokerReady="true" | ||
break | ||
fi | ||
done | ||
done | ||
|
||
# Now broker pod needs to be running | ||
echo | ||
print_blue "Waiting for broker pod with prefix ${brokerPrefix} to be running..." | ||
brokerReady="false" | ||
while [[ "${brokerReady}" == "false" ]]; do | ||
echo -n "." | ||
|
||
# TODO - we likely want to check for running OR completed, it's rare but sometimes they can complete too fast. | ||
for pod in $(kubectl get pods --namespace ${NAMESPACE} --field-selector=status.phase=Running --output=jsonpath='{.items[*].metadata.name}'); do | ||
if [[ "${pod}" == ${brokerPrefix}* ]]; then | ||
echo | ||
print_green "🌀️ Broker pod is running." | ||
brokerReady="true" | ||
break | ||
fi | ||
done | ||
done | ||
echo | ||
print_blue "Creating port forward to interact with RESTful API..." | ||
for pod in $(kubectl get pods --namespace ${NAMESPACE} --field-selector=status.phase=Running --output=jsonpath='{.items[*].metadata.name}'); do | ||
if [[ "${pod}" == ${brokerPrefix}* ]]; then | ||
run_echo kubectl port-forward -n flux-operator ${pod} 5000:5000 | ||
break | ||
fi | ||
done | ||
else | ||
echo | ||
print_blue "Creating port forward to interact with RESTful API..." | ||
for pod in $(kubectl get pods --namespace ${NAMESPACE} --field-selector=status.phase=Running --output=jsonpath='{.items[*].metadata.name}'); do | ||
if [[ "${pod}" == ${brokerPrefix}* ]]; then | ||
run_echo kubectl port-forward -n flux-operator ${pod} 5000:5000 | ||
break | ||
fi | ||
done | ||
print_green "🌀️ Found existing MiniCluster in ${NAMESPACE}" | ||
fi |