virtual_cluster_scheduler

#!/bin/bash

### Threshold variables ###
W1=0.1                                          #Weight 1 takes effect if queuing running ratio is lower than set threshold (qrr)
W2=0.3                                          #Weight 2 takes effect if running finished mean time ratio is larger than 1.0
W3=0.3                                          #Weight 3 takes effect if running finished mean time ratio is greater or equal than the set threshold (rfr) but lower equal than running finished difference (rfd1)
W4=0.2                                          #Weight 4 takes effect if running finished mean time ratio is smaler than rfr and smaller than running finished difference (in seconds) (rfd2)
W5=-0.2                                         #Weight 5 takes effect if no jobs are running or the running CPU capacity ratio is lower than the threshold (rCcr), NEGATIVE value !!!
qrr=0.4                                         #Threshold for the queueing running ratio value (#queueud jobs through #running jobs)
rfr=0.8                                         #Threshold for the running finished ratio value (consumed time of running jobs through used walltime of finished jobs staying in the queue)
rfd1=400                                        #Threshold 1 for the running finished difference in seconds 
rfd2=200                                        #Threshold 2 for the running finished difference in seconds
rfd1_multiplicator=200							#Multiplicator for rfd1 threshold to keep the range relative to the real walltimes instead of fixed
rfd2_multiplicator=50							#Multiplicator for rfd2 threshold to keep the range relative to the real walltimes instead of fixed
rCcr=0.2										#Threshold for the running CPU capacity ratio (#running jobs through max available CPUs)
cnc=2											#Threshold for the cluster node capacity (set value not lower than the initial number of cluster nodes)
mnc=9											#Threshold for the maximal number of nodes allowed (set value not higher than openstack quotas)
start_threshold=1.0								#Threshold for the start of a new node by the sum of weights (W1,W2,W3,W4,W5)
stop_threshold=-1								#Threshold for the stop of an existing node (downsizing the cluster)

### Logdate variables ###
month_short=$(date +%b)
day=$(date +%d)
timestamp=$(date +%T)
logtime="$month_short$day $timestamp"

###################################################################
######################### BEGIN ###################################
### Sum up weights and decide if new node is started or stopped ###
###################################################################
cluster_CPU_capacity_array=($(sudo cat /var/spool/torque/server_priv/nodes | awk -F "=" '{print $2}'))  #Get number of CPUs per node from torque file as array
cluster_CPU_capacity=0                                                                                  #Initialize number of CPUs variable
for i in "${cluster_CPU_capacity_array[@]}"                                                             #Iterate over CPU number array 
do
    cluster_CPU_capacity=$( echo $cluster_CPU_capacity + $i | bc -l)                                    #Sum up the number of CPUs currently available
done
echo "$logtime Max. number of cores available (free and used): " $cluster_CPU_capacity

cluster_nodes_capacity_array=($(sudo cat /var/spool/torque/server_priv/nodes | awk '{print $1}'))       #Get number of nodes from torque file as array
cluster_nodes_capacity=${#cluster_nodes_capacity_array[@]}						#Get length of array, equals number of nodes

#cluster_nodes_capacity=0                                                                                #Initialize number of nodes available variable
#for i in "${cluster_nodes_capacity_array[@]}"                                                           #Iterate over nodes number array
#do
#    cluster_nodes_capacity=$( echo $cluster_nodes_capacity + 1 | bc -l)                                 #Sum up the number of nodes currently available
#done
echo "$logtime Number of nodes: " $cluster_nodes_capacity


if [ -f /home/centos/.virtual_cluster_stats_start ]; then                                               #Check if .virtual_cluster_stats_start file exists
	sum_of_weights_array_start=($(cat /home/centos/.virtual_cluster_stats_start | awk '{print $2}'))    #Get starting weights from file into array
	sum_of_weights_start=0                                                                              #Initialize starting weights sum variable
	for i in "${sum_of_weights_array_start[@]}"                                                         #Iterate over starting weights array
	do
		sum_of_weights_start=$( echo $sum_of_weights_start + $i | bc -l)                                #Sum up the starting weights
	done
	
	echo "$logtime Current sum of weights: $sum_of_weights_start"
	
	if (( $(echo "$sum_of_weights_start <= 0" | bc -l) )); then                                         #Check if the starting weights sum is leq than 0
		rm -f /home/centos/.virtual_cluster_stats_start                                                 #Remove .virtual_cluster_stats_start file 
	else
		if (( $(echo "$sum_of_weights_start >= $start_threshold" | bc -l) )); then                      #Check if starting weights sum is higher than starting threshold
			if (( $cluster_nodes_capacity < $mnc )); then
				if [ -f /home/centos/.VALET_status ]; then
                	current_status=$(cat /home/centos/.VALET_status)
                	if [ "$current_status" == "blocked" ]; then
                    	echo "$logtime Current status is $current_status and therefore not cleared"
                    	exit 0;
					else
						echo "$logtime Start new node based on last timesteps as sum of weights ($sum_of_weights_start) is larger than threshold ($start_threshold)"	#Start new node
						echo "start" > /home/centos/.VALET_status												#(Over)Write "start" to file for VALET_balancer_executor on desktop
						rm -f /home/centos/.virtual_cluster_stats_start                                 		#Remove .virtual_cluster_stats_start as reset
						exit 0;
					fi
				else
					echo "$logtime Start new node based on last timesteps as sum of weights ($sum_of_weights_start) is larger than threshold ($start_threshold)"	#Start new node
					echo "start" > /home/centos/.VALET_status												#(Over)Write "start" to file for VALET_balancer_executor on desktop
					rm -f /home/centos/.virtual_cluster_stats_start                                 		#Remove .virtual_cluster_stats_start as reset
					exit 0;
				fi									    											#Stop execution of the script with 0 = success and skip the rest
			else
				echo "$logtime Sum of weights of last timesteps would be sufficient but maximal number of nodes ($mnc) is already reached."
				echo "stay" > /home/centos/.VALET_status												#(Over)Write "stay" to file for VALET_balancer_executor on desktop
			fi
		else
			echo "$logtime Sum of weights ($sum_of_weights_start) is lower than the threshold ($start_threshold). No new node will be added"
			echo "stay" > /home/centos/.VALET_status                                        			#(Over)Write "stay" to file for VALET_balancer_executor on desktop
		fi 
	fi
else                                                                                                    #If .virtual_cluster_stats_start file does not exist
	if [ -f /home/centos/.virtual_cluster_stats_stop ]; then                                            #Check if .virtual_cluster_stats stop file exists
		sum_of_weights_array_stop=($(cat /home/centos/.virtual_cluster_stats_stop | awk '{print $2}'))  #Get stopping weights from file into array
        	sum_of_weights_stop=0                                                                       #Initialize stopping weights sum variable
        	for i in "${sum_of_weights_array_stop[@]}"                                                  #Iterate over stopping weights array
        	do  
                sum_of_weights_stop=$( echo $sum_of_weights_stop + $i | bc -l)							#Sum up the stopping weights
        	done
	
		if (( $(echo "$sum_of_weights_stop <= $stop_threshold" | bc -l) )); then                        #Check if stopping weights sum is leq than threshold (negative values) 
			if (( $cluster_nodes_capacity > $cnc )); then
				if [ -f /home/centos/.VALET_status ]; then
                	current_status=$(cat /home/centos/.VALET_status)
                	if [ "$current_status" == "blocked" ]; then
                    	echo "$logtime Current status is $current_status and therefore not cleared"
                    	exit 0;                                           	#Check if cluster has more nodes than the initial cluster
                	else
                		rm -f /home/centos/.virtual_cluster_stats_stop                                  		#Remove .virtual_cluster_stats_stop file
						echo "$logtime Take away one node from the cluster as no jobs are currently running and more nodes than the initial cluster are present" #Remove one node from the cluster
						echo "stop" > /home/centos/.VALET_status                                        		#(Over)Write "stop" to file for VALET_balancer_executor on desktop
						exit 0;
					fi
				else
					rm -f /home/centos/.virtual_cluster_stats_stop                                  		#Remove .virtual_cluster_stats_stop file
					echo "$logtime Take away one node from the cluster as no jobs are currently running and more nodes than the initial cluster are present" #Remove one node from the cluster
					echo "stop" > /home/centos/.VALET_status                                        		#(Over)Write "stop" to file for VALET_balancer_executor on desktop
					exit 0;
					
				fi                                                                                 #Stop execution of the script with 0 = success and skip the rest
			else 
				echo "$logtime Initial cluster is already left. Cannot remove further nodes."          	#If initial cluster is already left. Do not remove any nodes
				rm -f /home/centos/.virtual_cluster_stats_stop											#Remove .virtual_cluster_stats_stop file as reset
				echo "stay" > /home/centos/.VALET_status                                        		#(Over)Write "stay" to file for VALET_balancer_executor on desktop
			fi
		else
			echo "$logtime Sum of weights ($sum_of_weights_stop) is not hitting the threshold ($stop_threshold). No node will be removed"

		fi
	fi
fi
###################################################################
######################### END #####################################
### Sum up weights and decide if new node is started or stopped ###
###################################################################


######################################
############## BEGIN #################
### Evaluate current cluster usage ###
######################################
queued_jobs=$(sudo env "PATH=$PATH" qstat -tn1 | awk '{print $10}' | grep -c Q)     							#Get number of queued jobs from torque
running_jobs=$(sudo env "PATH=$PATH" qstat -tn1 | awk '{print $10}' | grep -c R)    							#Get number of running jobs from torque
running_jobs_ids=($(sudo env "PATH=$PATH" qstat -tn1 | awk '{print $1,$10}' | grep R | awk '{print $1}'))       #Collect all job ids of running jobs in array

total_current_used_cores=0                                              #Initialize sum variable
for i in "${running_jobs_ids[@]}"                                       #Iterate over running job ids array
do
        if [[ $i != "Req'd" ]]; then                                    #Skip "Req'd" array entry
                job_stats=$(qstat -f $i)								#Save job information in variable
                if ! grep -q ppn <<< "$job_stats"; then         		#Check if no cores are given in the job script (per default one core is used)
                        core_count=1
                        total_current_used_cores=$( echo $total_current_used_cores + $core_count | bc -l)	#Add new cores to already summed up ones
                else																						#If ppn:X is given parse out numbers of nodes and cores
                        node_count=$(echo "$job_stats" | grep "Resource_List.nodes" | awk -F = '{print $2}'| awk -F: '{print $1}' | tr -d " \t\n\r")    #Get node count of job
                        core_count=$(echo "$job_stats" | grep "Resource_List.nodes" | awk -F:ppn= '{print $2}' | tr -d " \t\n\r")

                        if [[ $node_count != 1 ]]; then                                 #Check if node count is not 1, e.g. for MPI jobs
                                core_count=$(echo "$core_count * $node_count" | bc -l)  #Multiplie cores with number of nodes                   
                        fi
        
                        total_current_used_cores=$( echo $total_current_used_cores + $core_count | bc -l)       #Add new cores to already summed up ones
                fi
        fi  
done

echo "$logtime Number of currently used cores: $total_current_used_cores / $cluster_CPU_capacity"

if [[ $queued_jobs == 0 || $running_jobs == 0 ]]; then								#Check if one of the variables above is zero
	echo "$logtime Nothing to do, number of queued jobs or running jobs is zero"	#If one check is zero there is no need to add a new node
	echo "stay" > /home/centos/.VALET_status                                        #(Over)Write "stay" to file for VALET_balancer_executor on desktop

#	running_CPU_capacity_ratio=$(echo $running_jobs / $cluster_CPU_capacity | bc -l)			#Calculate ratio from runnign jobs through max number of CPUs
	running_CPU_capacity_ratio=$(echo $total_current_used_cores / $cluster_CPU_capacity | bc -l)	#Calculate ratio from currently used CPUs through max number of CPUs
	echo "$logtime Running CPU capacity ratio: $running_CPU_capacity_ratio"
	if (( $running_jobs == 0 || $(echo "$running_CPU_capacity_ratio < $rCcr" | bc -l) )); then	#Check if running jobs are 0 or ratio from above is lower than threshold
		if [ -f /home/centos/.virtual_cluster_stats_start ]; then								#If this is the case check if .virtual_cluster_stats_start file exists
			echo "W5 $W5" >> /home/centos/.virtual_cluster_stats_start							#Add weight to .virtual_cluster_stats_start file
			echo "$logtime W5 is added due to running jobs: $running_jobs or running CPU capacity hit threshold: $rCcr"
			exit 0;
		else                                                                                    #If .virtual_cluster_stats_start file does not exist
			if [ -f /home/centos/.virtual_cluster_stats_stop ]; then                            #Check if .virtual_cluster_stats_stop file exists
				echo "W5 $W5" >> /home/centos/.virtual_cluster_stats_stop                       #Add weight to .virtual_cluster_stats_stop file
				echo "$logtime W5 is added"
				exit 0;
			else                                                                                #If .virtual_cluster_stats_stop file does not exist
				echo "W5 $W5" > /home/centos/.virtual_cluster_stats_stop                        #Create new .virtual_cluster_stats_stop file and add weight to it
				echo "$logtime W5 is added"
				exit 0;
			fi
		fi
	else
		echo "$logtime Number of queued jobs are 0 but cluster is still used to its capacity"
	fi
else                                                                    
	queued_running_ratio=$(echo $queued_jobs / $running_jobs | bc -l)               #If queued or running jobs are not 0, calculate the ratio of #queued and #running jobs 

	echo "$logtime Number of queued jobs: "$queued_jobs
	echo "$logtime Number of running jobs: "$running_jobs
	echo "$logtime Ratio of queued to running jobs: " $queued_running_ratio

	if (( $(echo "$queued_running_ratio >= $qrr" | bc -l) )); then                  #Check if ratio is above the chosen threshold, new node may be helpful
		echo "$logtime Checking possibility to start new node as half or more jobs than the available capacity is queued"  
		finished_jobs_ids=($(sudo env "PATH=$PATH" qstat -tn1 | awk '{print $1,$10}' | grep C | awk '{print $1}' )) #Get job ids of finished jobs
		running_jobs_ids=($(sudo env "PATH=$PATH" qstat -tn1 | awk '{print $1,$10}' | grep R | awk '{print $1}'))   #Get job ids of running jobs

		number_finished_jobs=${#finished_jobs_ids[@]}                               #Get number of finished jobs
		number_running_jobs=$( expr ${#running_jobs_ids[@]} - 1 )                   #Get number of running jobs. Substract -1 as first entry is matching string "Req'd"

		sum_finished=0                                                              #Initialize sum of the walltime used by finished jobs
		total_finished=0                                                            #Initialize counter for the divisor of the sum (mean calculation)

		if [[ $number_finished_jobs == 0 ]]; then                                   #Check if #finished jobs is zero
			ratio_mean_finished=0.1                                             	#Set low pseudo count
		else                                    
			for i in "${finished_jobs_ids[@]}"                                  	#Iterate over finished job ids
			do
				total_walltime=$(sudo env "PATH=$PATH" qstat -f $i | grep total_runtime | awk '{print $3}') #Get walltime of the job
				sum_finished=$(echo $sum_finished + $total_walltime | bc -l)                                #Add to time sum 
				total_finished=$(( $total_finished + 1 ))                                                   #Increase counter
			done
			ratio_mean_finished=$(echo $sum_finished / $total_finished | bc -l)     #Calculate the mean value of the walltimes
			echo "$logtime Mean time finished: " $ratio_mean_finished
			
			predicted_time_overall_queued=$(echo "$queued_jobs * $ratio_mean_finished" | bc -l)	#Predicted time until all queueud jobs would be finished
			echo "$logtime The predicted time (seconds) until all queued jobs are finished: $predicted_time_overall_queued"
			
			rfd1=$(echo "$ratio_mean_finished * $rfd1_multiplicator" | bc -l)		#Dynamic rfd1 threshold calculation step
			echo "$logtime New rfd1 threshold in seconds: $rfd1"

			rfd2=$(echo "$ratio_mean_finished * $rfd2_multiplicator" | bc -l)		#Dynamic rfd2 threshold calculation step
			echo "$logtime New rfd2 threshold in seconds: $rfd2"

		fi

		sum_running=0                                                               #Initialize sum of the walltime used by running jobs
		total_running=0                                                             #Initialize counter for the divisor of the sum (mean calculation)

		if [[ $number_running_jobs == 0 ]]; then                                    #Check if number of running jobs is 0
			ratio_mean_running=0                                                    #Set ratio mean running directly to 0
		else                                                                        #If number of running jobs is unequal 0
			for i in "${running_jobs_ids[@]}"                                       #Iterate over running job ids array
			do
        			if [[ $i != "Req'd" ]]; then                                    #Skip "Req'd" array entry
        				used_walltime=$(sudo env "PATH=$PATH" qstat -f $i | grep resources_used.walltime | awk '{print $3}')    #Get walltime of the job
        				used_walltime_in_seconds=$(echo $used_walltime | awk -F: '{ print ($1 * 3600) + ($2 * 60) + $3 }')      #Convert time format to seconds
        				sum_running=$( echo $sum_running + $used_walltime_in_seconds | bc -l)                                   #Add to time sum
        				total_running=$(( $total_running + 1 ))                                                                 #Increase counter                                                 
        			fi  
			done
		fi
		ratio_mean_running=$(echo $sum_running / $total_running | bc -l)							#Calculate the mean value of the walltimes
		echo "$logtime Mean time running: " $ratio_mean_running
		echo "$logtime Mean time finished:" $ratio_mean_finished

		running_finished_ratio=$(echo $ratio_mean_running / $ratio_mean_finished | bc -l)			#Calculate ratio of mean time values of running and finished jobs

		echo "$logtime Mean time ratio: " $running_finished_ratio

		if (( $(echo "$running_finished_ratio > 1.0" | bc -l) )); then								#Check if time ratio is greater than 1
			echo "$logtime There should be soon some free slots. Wait to start new node"
			if [ -f /home/centos/.virtual_cluster_stats_start ]; then								#Further check if .virtual_cluster_stats_start file exists
				if (( $cluster_nodes_capacity < $mnc )); then
					echo "W2 $W2" >> /home/centos/.virtual_cluster_stats_start							#If it exists add weight to .virtual_cluster_stats_start file
					echo "$logtime W2 is added"
					exit 0;
				else
					echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W2 is not added"
				fi
			else
				if (( $cluster_nodes_capacity < $mnc )); then
					echo "W2 $W2" > /home/centos/.virtual_cluster_stats_start 	                  		#If it does not exist create the file and add weight
					echo "$logtime weight W2 is added"
					if [ -f /home/centos/.virtual_cluster_stats_stop ]; then                		    #Check if .virtual_cluster_stats_stop file exists
						rm -f /home/centos/.virtual_cluster_stats_stop              		            #Remove .virtual_cluster_stats_stop file as reset
						exit 0;
					fi
				else
					echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W2 is not added"
				fi
			fi
		else											                		                    #Else, if running_finished_ratio is lower than 1
			running_finished_difference=$(echo $ratio_mean_finished - $ratio_mean_running | bc -l)  #Calculate difference between mean times finished and running
			echo "$logtime Mean time difference of finished to running: " $running_finished_difference
			if (( $(echo "$running_finished_ratio >= $rfr" | bc -l) )); then                        #Check if time ratio is greater/equal than threshold (rfr)
#				if (( $(echo "$running_finished_difference <= $rfd1" | bc -l) )); then              #Check if time difference is lower than threshold (rfd1), no new node will start
				if (( $(echo "$predicted_time_overall_queued <= $rfd1" | bc -l) )); then 			#Check if predicted waiting time is greater/equal than rfd1, add new node
					echo "$logtime There should be soon some free slots. Wait to start new node"
					if [ -f /home/centos/.virtual_cluster_stats_start ]; then                       #Check if .virtual_cluster_stats_start file exists
						if (( $cluster_nodes_capacity < $mnc )); then
							echo "W3 $W3" >> /home/centos/.virtual_cluster_stats_start                  #Add weight to .virtual_cluster_stats_start file
							echo "$logtime W3 is added"
							exit 0;
						else
							echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W3 is not added"
						fi
					else                                                                            #If .virtual_cluster_stats_start file does not exist
						if (( $cluster_nodes_capacity < $mnc )); then
							echo "W3 $W3" > /home/centos/.virtual_cluster_stats_start                   #Create virtual_cluster_stats_start file and add the weight
							echo "$logtime W3 is added"
						else
							echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W3 is not added"
						fi 
						if [ -f /home/centos/.virtual_cluster_stats_stop ]; then                    #Check if virtual_cluster_stats_stop file exists
							rm -f /home/centos/.virtual_cluster_stats_stop                      	#Remove virtual_cluster_stats_stop file as reset
							exit 0;
						fi
					fi
				else                                                                                #Else difference is larger than threshold rfd1, add new node
					if (( $cluster_nodes_capacity < $mnc )); then
						if [ -f /home/centos/.VALET_status ]; then
                			current_status=$(cat /home/centos/.VALET_status)
                			if [ "$current_status" == "blocked" ]; then
                    			echo "$logtime Current status is $current_status and therefore not cleared"
                    			exit 0;
 							else
								echo "$logtime Running finished ratio is larger then threshold of $rfr and predicted time is larger than $rfd1 s. Start new node"
								echo "start" > /home/centos/.VALET_status									#(Over)Write "start" to file for VALET_balancer_executor on desktop
								exit 0;
							fi
						else
							echo "$logtime Running finished ratio is larger then threshold of $rfr and predicted time is larger than $rfd1 s. Start new node"
							echo "start" > /home/centos/.VALET_status									#(Over)Write "start" to file for VALET_balancer_executor on desktop
							exit 0;
						fi
					else
						echo "$logtime Running finished ratio is larger $rfr and predicted time is larger than $rfd1 but maximal number of nodes ($mnc) is already reached"
						echo "stay" > /home/centos/.VALET_status                                    #(Over)Write "stay" to file for VALET_balancer_executor on desktop
						exit 0;
					fi 
				fi
			else                                                                                    #Else, if running finished ratio is lower than threshold (rfr)
#				if (( $(echo "$running_finished_difference >= $rfd2" | bc -l) )); then              #Check if time difference is greater/equal than rfd2, add new node
				if (( $(echo "$predicted_time_overall_queued >= $rfd2" | bc -l) )); then			#Check if predicted waiting time is greater/equal than rfd2, add new node
					if  (( $cluster_nodes_capacity < $mnc )); then
						if [ -f /home/centos/.VALET_status ]; then
                			current_status=$(cat /home/centos/.VALET_status)
                			if [ "$current_status" == "blocked" ]; then
                    			echo "$logtime Current status is $current_status and therefore not cleared"
                    			exit 0;
                    		else
        						echo "$logtime The predicted time is larger than $rfd2. There will not be soon some free slots. Start new node."
								echo "start" > /home/centos/.VALET_status									#(Over)Write "start" to file for VALET_balancer_executor on desktop
								exit 0;
							fi
						else
							echo "$logtime The predicted time is larger than $rfd2. There will not be soon some free slots. Start new node."
							echo "start" > /home/centos/.VALET_status									#(Over)Write "start" to file for VALET_balancer_executor on desktop
							exit 0;
						fi
					else
						echo "$logtime The predicted time is larger than $rfd2 but the maximal number of nodes ($mnc) is already reached."
						echo "stay" > /home/centos/.VALET_status									#(Over)Write "stay" to file for VALET_balancer_executor on desktop
						exit 0;
					fi
				else                                                                                #Else, if time difference is lower than rfd2, no new node
					echo "$logtime Ratio is smaller than threshold of $rfr but predicted is smaller than $rfd2 s. No new node will be started "
					if [ -f /home/centos/.virtual_cluster_stats_start ]; then                       #Check if virtual_cluster_stats_start file exists
						if (( $cluster_nodes_capacity < $mnc )); then
							echo "W4 $W4" >> /home/centos/.virtual_cluster_stats_start					#Add weight to .virtual_cluster_stats_start file
							echo "$logtime W4 is added"
							exit 0;
						else
							echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W4 is not added"
						fi
					else                                                                            #If .virtual_cluster_stats_start file does not exist
						if (( $cluster_nodes_capacity < $mnc )); then
							echo "W4 $W4" > /home/centos/.virtual_cluster_stats_start					#Create virtual_cluster_stats_start file and add the weight
							echo "$logtime W4 is added"
						else
							echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W4 is not added"
						fi
						if [ -f /home/centos/.virtual_cluster_stats_stop ]; then					#Check if virtual_cluster_stats_stop file exists
							rm -f /home/centos/.virtual_cluster_stats_stop							#Remove virtual_cluster_stats_stop file as reset
							exit 0;
						fi
					fi
				fi
			fi
		fi
	else                                                                                            #Else, if queued running ratio is lower then threshold (qrr), no new node is started
		echo "$logtime No new node needs to be added, capacity is sufficient"
		echo "stay" > /home/centos/.VALET_status                                        			#(Over)Write "stay" to file for VALET_balancer_executor on desktop
		if [ -f /home/centos/.virtual_cluster_stats_start ]; then                                   #Check if virtual_cluster_stats_start file exists
			if (( $cluster_nodes_capacity < $mnc )); then
				echo "W1 $W1" >> /home/centos/.virtual_cluster_stats_start                              #Add weight to .virtual_cluster_stats_start file
				echo "$logtime W1 is added"
				exit 0;
			else
				echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W1 is not added"
			fi
		else                                                                                        #If .virtual_cluster_stats_start file does not exist
			if (( $cluster_nodes_capacity < $mnc )); then                              
				echo "W1 $W1" > /home/centos/.virtual_cluster_stats_start                               #Create virtual_cluster_stats_start file and add the weight
				echo "$logtime W1 is added"
			else
				echo "$logtime Maximal number of nodes ($mnc) is already reached, weight W1 is not added"
			fi
			if [ -f /home/centos/.virtual_cluster_stats_stop ]; then                                #Check if virtual_cluster_stats_stop file exists
				rm -f /home/centos/.virtual_cluster_stats_stop                                      #Remove virtual_cluster_stats_stop file as reset
				exit 0;
			fi
		fi
	fi
fi
######################################
############## END ###################
### Evaluate current cluster usage ###
######################################