Create /galaxy and /galaxy-repl symlinks on Jetstream for old TS depe…

…ndency resolution, plus log suspends/resumes seperately by hosts in play

roles/slurmscale/files/group_vars/all/cluster.yml

@@ -31,25 +31,22 @@ jetstream_nfs_filesystems:
     dir: scratch0
 
 directories:
-  #- path: /galaxy-repl/test
-  #- path: /galaxy-repl/main
+  - path: /galaxy-repl/test
+    mode: "0755"
+  - path: /galaxy-repl/main
+    mode: "0755"
   - path: /var/lib/slurm
     mode: "0755"
 
 # symlinks for deps must exist due to path issues
-#links:
-#  - path: /galaxy
-#    src: /galaxy-repl
-#    force: yes
-#  - path: /galaxy-repl/test/deps
-#    src: /cvmfs/test.galaxyproject.org/deps
-#    force: yes
-#  - path: /galaxy-repl/test/tool_data
-#    src: /cvmfs/data.galaxyproject.org/byhand/location
-#    force: yes
-#  - path: /galaxy-repl/main/deps
-#    src: /cvmfs/main.galaxyproject.org/deps
-#    force: yes
-#  - path: /galaxy-repl/main/tool_data
-#    src: /cvmfs/data.galaxyproject.org/byhand/location
-#    force: yes
+links:
+  - dest: /galaxy
+    src: /galaxy-repl
+  - dest: /galaxy-repl/test/deps
+    src: /cvmfs/test.galaxyproject.org/deps
+  - dest: /galaxy-repl/test/tool_data
+    src: /cvmfs/data.galaxyproject.org/byhand/location
+  - dest: /galaxy-repl/main/deps
+    src: /cvmfs/main.galaxyproject.org/deps
+  - dest: /galaxy-repl/main/tool_data
+    src: /cvmfs/data.galaxyproject.org/byhand/location

roles/slurmscale/files/roles/galaxynodeconfig/tasks/main.yml

@@ -59,6 +59,14 @@
         state: directory
       with_items: "{{ directories }}"
 
+    - name: Create links
+      file:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        state: link
+        force: yes
+      with_items: "{{ links }}"
+
   rescue:
 
     - name: Destroy instance on failure
@@ -67,6 +75,7 @@
         name: "{{ inventory_hostname }}"
         state: absent
       delegate_to: localhost
+      become: no
 
     - name: Fail due to previous failure
       fail:

roles/slurmscale/tasks/main.yml

@@ -54,6 +54,12 @@
     - etc/ansible/roles/galaxynodeconfig
     - etc/ansible/roles/galaxynodeconfig/tasks
 
+- name: Create slurm scale log directory
+  file:
+    path: "/var/log/slurm/elastic"
+    state: directory
+    owner: "{{ slurm_scale_user }}"
+
 - name: Copy ansible files
   copy:
     src: "{{ item.name }}"

roles/slurmscale/templates/suspend_resume.sh.j2

@@ -1,11 +1,15 @@
 #!/bin/bash
 
 log_loc=/var/log/slurm/elastic.log
+log_dir=/var/log/slurm/elastic
 
 . {{ slurm_scale_root }}/bin/activate
 
 cd {{ slurm_scale_root }}/etc/ansible
 
+echo "Node $op invoked: $0 $*" >> $log_loc
+echo "Start: $(date)" >> $log_loc
+
 case $(basename $0) in
     slurm_suspend)
         op='suspend'
@@ -23,11 +27,15 @@ case $(basename $0) in
         ;;
 esac
 
-echo "Node $op invoked: $0 $*" >> $log_loc
-
 names=$(scontrol show hostname "$1" | xargs echo | tr ' ' ',')
 
+echo "Start: $(date)" >> "${log_dir}/${names}.log"
+start=$(date +%s)
 set -x
-{ ansible-playbook -l "$names" ${op}.yml; } >> $log_loc 2>&1
+{ ansible-playbook -l "$names" ${op}.yml; } >> "${log_dir}/${names}.log" 2>&1
 { [ -f inventory/spawn.$$ ] && rm -f inventory/spawn.$$; } >> $log_loc 2>&1
 { set +x; } 2>/dev/null
+echo "End: $(date)" >> "${log_dir}/${names}.log"
+echo "Elapsed: $(($(date +%s) - $start))" >> "${log_dir}/${names}.log"
+
+echo "End: $(date)" >> $log_loc