Skip to content

Commit

Permalink
Infrastructure for new Galaxy job handler VMs
Browse files Browse the repository at this point in the history
  • Loading branch information
natefoo committed Aug 11, 2016
1 parent d8dec91 commit 3851a8e
Show file tree
Hide file tree
Showing 6 changed files with 184 additions and 6 deletions.
Binary file added roles/cvmfs/files/cvmfs_wipecache.centos_7
Binary file not shown.
22 changes: 22 additions & 0 deletions taccenv/group_vars/slurmclients.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---

group_groups:
- name: slurm
gid: 40302

group_users:
- name: slurm
group: slurm
comment: 'Slurm Workload Manager'
uid: 40302
system: yes
home: /var/lib/slurm
shell: /bin/bash

controller_name: galaxy02
controller_ip: 129.114.60.174

slurm_yum_repo_baseurl: https://depot.galaxyproject.org/yum/el/$releasever/$basearch

slurmd_spool_dir: /var/tmp/slurmd.spool
slurmctld_state_dir: /tmp/slurm.state
13 changes: 11 additions & 2 deletions taccenv/inventory
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,18 @@ maingalaxyservers
galaxy01.tacc.utexas.edu

[maingalaxyservers]
[maingalaxyservers:children]
maingalaxywebservers
maingalaxyjobservers

[maingalaxywebservers]
galaxy-web-01.tacc.utexas.edu
galaxy-web-02.tacc.utexas.edu

[maingalaxyjobservers]
galaxy-web-03.tacc.utexas.edu
galaxy-web-04.tacc.utexas.edu

[galaxynodes]
roundup[49:64].tacc.utexas.edu

Expand All @@ -33,6 +42,6 @@ galaxy06.tacc.utexas.edu
galaxynodes
galaxyservers

[jetstreamnfsclients]
[jetstreamnfsclients:children]
[slurmclients]
[slurmclients:children]
galaxyservers
28 changes: 24 additions & 4 deletions taccenv/playbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,24 @@
tags: system,services
tags: initial

- name: Forward mail for local accounts
hosts: all
remote_user: root
tasks:
- name: Forward mail for {{ item }}
lineinfile:
dest: /etc/aliases
regexp: "^{{ item }}:"
line: "{{ item }}: root@galaxyproject.org"
with_items:
- g2test
- g2main
- postgres
notify: run newaliases
handlers:
- name: run newaliases
command: /usr/bin/newaliases

- name: Tasks for webservers
hosts: webservers
remote_user: root
Expand Down Expand Up @@ -79,9 +97,11 @@
- role: cvmfs
tags: cvmfs

- name: Tasks for Jetstream NFS clients
hosts: jetstreamnfsclients
- name: Tasks for slurm hosts
hosts: slurmservers:slurmclients
remote_user: root
vars_files:
- secret_group_vars/slurmclients.yml
roles:
- role: jetstream_nfs
tags: nfs
- role: slurm
tags: slurm
78 changes: 78 additions & 0 deletions taccenv/secret_group_vars/slurmclients.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
$ANSIBLE_VAULT;1.1;AES256
33373432336635343338383932623237333231643935616539323365366637366530363735646265
6231393937323935356131383739383631316132353038310a356138303838326134353661393130
64313939336130376364643438333939376233666633333133653765363336353631313830393664
6237643238316533320a363434323466333932343062633638323962363366656264353062656661
38383532373731343734323430383563373064316530613065396436646237393132333730363063
66633738656461643333343734353963303236616432636363343564336336343235343739626534
30656532326532353966383333393933663465646333303236336231333765363665356264363732
38613137376138666531343034613031383637633033633463323232353136653365653861336530
62623639646431663038323161663334636133326533613836396461613839386533643638393263
62373536633562323236323531303237303630356633653438663234646165303764626364396432
62313339353035623430626335616537626139336431363662313330366464653839646132376633
63653261396265303138336133346466346233323239393461656135323939396230316535353434
38343461343665363832323166383333383036633739643665623731373263616166613030396532
31656565626463346636666330323161323034636237663738336466623463643131343136333066
64376666626131616265663961633262663135373737306437323330353264336637343731316637
64386666323634393939306462616339356632623830643765366533663638376237383562623838
64663665626632303633383238623033633736366439636566353336656232326331633265303064
37313363393735653765646139626334623935396563663836663235643762393564623436633965
38653636633936336261376538393036636331323135643332666531616130653962303132656332
61633765313533616164333536643431616462666366653537653062303165653762393731366362
63383834663561656639316161666264626563376637633533343434383062373361356561656236
31306232316636656133373164306236323764363635393031343063393830396631396262626161
61633931313732393332623165623335336339393234656232396634613463373134306237373164
66626465666566613530663937353932653430376431356461306533316263643163656333616234
33383066383162376139333935393064363339366366336239376539653866366335343462613430
64313161613131393264663162636133363032643562336433346338636137386661336333343437
30383064613339323936623730666533336361616533663739343262383833333334633339373638
61613335306466613166313734623631313737353331336438396266633637393432346162633264
39356565373465313539343637343032653661306230353865353261666665646231386161363464
35336566633065343766376235663866363131626532643966643630653934333631313163353230
38336432323230326338303833316431633564383039613766666433316132303935366430353231
37373362383439666631363937353262353562626666656462326639323334656261336439353734
61313866326665343235326237306434366261363032326662383130656637646361316366346136
63323837376162623362313863666433623366393963383039653962396537333666633563326166
33643461363262636237316139616666656433646430386663303439336530643064633266653663
61323732323639303638326466643930633561356434353332643832323130663262643336373331
38663130353031363462373039333530393637623163333765393831646637346637666334613261
65633234363936363131303062363139636139643666376466623064346532653833623366363763
61373038363739363639396165326135383136613933326463346632333461636637633163383234
64373663313966666563303930313666626565616432333534373130386163666139366363366533
32636230366365376636366235666165323533303233646332613663666166356539326330636661
66616339336439656231363864383532326331363835383731333935383464343464386438656133
38373532643530386133653931626465646532393739353130663631386466633236346631663031
30383464643865656638663933333335653265303964376439376563343465646630373936613439
33663536616662613762363836383731376434383761316232303331336231623662633563616636
36306163666137396135353338333939386565303064333739616238633237616536663237323131
65356165363261303463366461643831646138303464323137393866336432646238656263376133
63633439326665313166343134306137396363363163623331633963363666623531613735343635
39303466623763616431306130376565616635313034386331326135633338316234636331366564
32333639343735373462373735663334373337336434313762316439653139326537353764656537
38633934313765663038333335376134356237356331643963306665333061356366653665313237
61616565333831356362323536313435343637396137353636663631373965623361666462363063
36653637316635343031323232646332316361366639353730613034336336643962306635653530
35613761653564316164663335633564373634386637343139666162636331313566623637623334
61383364643334666163323262363863613361653162303535646438306364633730363337383836
38343535393963633166376161376564623731326236646231656162613366643432386364616162
32656230633266333562663937336230303039373164393464633766313166386436656165313832
33643736656333323638333138633265366536383138646331613035643263643131356136343764
61643961313330653264616337323733663165653538343334663933326330613939633265316364
33666632323631633263353964613231363966393266373836356438383262623137363632656239
32393364393461343436646338383264623634366132626239373031653235313937303738313861
63356431363637383338343736616132616337353164613434366531303230636136646631623264
32623933636161356164323036383863336434323938343365343933623533363639386531363636
62613833346638653231663139336461363137616335346430613932353838363762646637626538
61353132386564306465393365313766626633326463316636316465633161613632373264633562
65393438613635373435383666303961643335656362336161306163616432383061303235633831
30376535313830643832646336323139393030663730326239623332353833343464393133303832
31666230666462626334633464333438623766663361353064646164363336646535333066313562
36366435643862366262373531343463333064656135656264346262383335343039343536653765
38666266623930386562623934353531373164636538356231336362393962386639643835316438
39393834353231336533316535376230303438326664646237343130663836653064613031343766
35366331313932383336636434356337376332663435636530376538366330396530363036346531
65323164633937336332623636313330303136393932646661343762396365356165653132356463
64663266656336303838303334336536666230623963366230366630653135373139383135646539
63366562636561346461393239363739643862623338323439303465346362346532626330346636
36376165393436393565323938666132326136333136343030356462363338343037636139393665
3565
49 changes: 49 additions & 0 deletions taccenv/templates/slurm/slurm.conf.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
##
## This file is maintained by Ansible - CHANGES WILL BE OVERWRITTEN
##
ControlMachine={{ controller_name }}
ControlAddr={{ controller_ip }}
#
AuthType=auth/munge
FastSchedule=1
JobCompLoc=/var/log/slurm/slurm.job.log
JobCompType=jobcomp/filetxt
PluginDir=/usr/lib64/slurm
SchedulerType=sched/backfill
SelectType=select/cons_res
SelectTypeParameters=CR_CPU_Memory
SlurmUser=slurm
SlurmctldPort=7002
SlurmctldTimeout=300
SlurmdPort=7003
SlurmdSpoolDir={{ slurmd_spool_dir }}
SlurmdTimeout=300
StateSaveLocation={{ slurmctld_state_dir }}
SwitchType=switch/none
TreeWidth=50
#AccountingStorageType=accounting_storage/filetxt
DefaultStorageLoc=/var/log/slurm/slurm_accounting
AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageHost=galaxy02
#AccountingStoragePort=6819
AccountingStoragePort=30001
JobAcctGatherType=jobacct_gather/linux
JobAcctGatherFrequency=task=15
ProctrackType=proctrack/linuxproc
ClusterName=jetstream-tacc
ReturnToService=1

#
# Node Configurations
#
NodeName=roundup[49-64] RealMemory=122880 CPUS=24 State=UNKNOWN

#
# Partition Configurations
#
PartitionName=multi State=UP MaxTime=48:20:00 MaxNodes=1 DefMemPerCPU=5120 Nodes=roundup[49-60] LLN=YES
PartitionName=normal State=UP MaxTime=48:20:00 MaxNodes=1 DefMemPerCPU=7680 Nodes=roundup[61-64] LLN=YES Default=YES
PartitionName=reserved State=UP MaxTime=48:20:00 MaxNodes=1 DefMemPerCPU=7680 Nodes=roundup[49-64]

SlurmctldDebug=6
SlurmdDebug=6

0 comments on commit 3851a8e

Please sign in to comment.