From 74f824f930caae83ded38a19919f0711ab2786e3 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 2 Jun 2021 14:35:52 +0200 Subject: [PATCH] support reports, mules --- defaults/main.yml | 16 ++++ tasks/main.yml | 27 +++++++ tasks/systemd-mule.yml | 24 ++++++ tasks/systemd-reports.yml | 23 ++++++ tasks/systemd-zerg.yml | 78 +++++++++++++++++++ templates/systemd/galaxy-handler@.service.j2 | 23 ++++++ templates/systemd/galaxy-reports.service.j2 | 23 ++++++ .../galaxy-workflow-scheduler@.service.j2 | 23 ++++++ templates/systemd/galaxy-zergling@.service.j2 | 24 ++++++ templates/systemd/galaxy-zergpool.service.j2 | 23 ++++++ templates/systemd/galaxy.service.j2 | 23 ++++++ templates/systemd/resource_controls.conf.j2 | 31 ++++++++ 12 files changed, 338 insertions(+) create mode 100644 tasks/systemd-mule.yml create mode 100644 tasks/systemd-reports.yml create mode 100644 tasks/systemd-zerg.yml create mode 100644 templates/systemd/galaxy-handler@.service.j2 create mode 100644 templates/systemd/galaxy-reports.service.j2 create mode 100644 templates/systemd/galaxy-workflow-scheduler@.service.j2 create mode 100644 templates/systemd/galaxy-zergling@.service.j2 create mode 100644 templates/systemd/galaxy-zergpool.service.j2 create mode 100644 templates/systemd/galaxy.service.j2 create mode 100644 templates/systemd/resource_controls.conf.j2 diff --git a/defaults/main.yml b/defaults/main.yml index ec336a8..5bd5bfd 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -19,6 +19,9 @@ galaxy_fetch_dependencies: yes galaxy_build_client: yes galaxy_manage_errordocs: no galaxy_backup_configfiles: yes +galaxy_manage_systemd: no # For Galaxy +galaxy_manage_systemd_reports: no # For Reports + # Control whether to output verbose task diffs (e.g. when performing a git update) when running Ansible with --diff. # Only suppresses diff on tasks that are likely to be verbose, not all tasks @@ -274,3 +277,16 @@ galaxy_client_build_steps: - stageLibs - plugins + +# systemd options +galaxy_systemd: + env: "" #Any extra env vars + mode: mule + memory_limit: + mule: 16 + reports: 5 + #workflow: 5 + #zergpool: 8 + #zergling: 16 + #handler: 5 + stats: 4001 # Port for stats to listen on. diff --git a/tasks/main.yml b/tasks/main.yml index f7beb99..7a13166 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -120,3 +120,30 @@ when: galaxy_manage_errordocs tags: - galaxy_manage_errordocs + +#- name: Include systemd unit setup tasks (Zerg Mode) + #include_tasks: + #file: systemd-zerg.yml + #apply: + #tags: galaxy_manage_systemd + #when: galaxy_systemd.mode == "zerg" + #tags: + #- galaxy_manage_systemd + +- name: Include systemd unit setup tasks (Mules) + include_tasks: + file: systemd-mule.yml + apply: + tags: galaxy_manage_systemd + when: galaxy_systemd.mode == "mule" + tags: + - galaxy_manage_systemd + +- name: Include systemd unit setup tasks (Reports) + include_tasks: + file: systemd-reports.yml + apply: + tags: galaxy_manage_systemd_reports + when: galaxy_manage_systemd_reports + tags: + - galaxy_manage_systemd_reports diff --git a/tasks/systemd-mule.yml b/tasks/systemd-mule.yml new file mode 100644 index 0000000..37b4369 --- /dev/null +++ b/tasks/systemd-mule.yml @@ -0,0 +1,24 @@ +--- + +- name: Manage Paths + block: + + - name: Deploy Galaxy Unit + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy.service.j2 + dest: /etc/systemd/system/galaxy.service + notify: + - daemon reload + - galaxy mule start + + - name: Enable Galaxy Unit + systemd: + name: galaxy.service + enabled: yes + + remote_user: "{{ galaxy_remote_users.root | default(__galaxy_remote_user) }}" + become: "{{ true if galaxy_become_users.root is defined else __galaxy_become }}" + become_user: "{{ galaxy_become_users.root | default(__galaxy_become_user) }}" diff --git a/tasks/systemd-reports.yml b/tasks/systemd-reports.yml new file mode 100644 index 0000000..a225b56 --- /dev/null +++ b/tasks/systemd-reports.yml @@ -0,0 +1,23 @@ +--- + +- name: Manage Paths + block: + + - name: Deploy Galaxy Reports unit + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy-reports.service.j2 + dest: /etc/systemd/system/galaxy-reports.service + notify: daemon reload + + - name: Enable reports and ensure it is running + systemd: + name: galaxy-reports.service + enabled: yes + state: started + + remote_user: "{{ galaxy_remote_users.root | default(__galaxy_remote_user) }}" + become: "{{ true if galaxy_become_users.root is defined else __galaxy_become }}" + become_user: "{{ galaxy_become_users.root | default(__galaxy_become_user) }}" diff --git a/tasks/systemd-zerg.yml b/tasks/systemd-zerg.yml new file mode 100644 index 0000000..32e85b6 --- /dev/null +++ b/tasks/systemd-zerg.yml @@ -0,0 +1,78 @@ +--- + +- name: Manage Paths + block: + + - name: Deploy Zergpool unit + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy-zergpool.service.j2 + dest: /etc/systemd/system/galaxy-zergpool.service + notify: daemon reload + + - name: Deploy Zergling units + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy-zergling@.service.j2 + dest: /etc/systemd/system/galaxy-zergling@.service + notify: daemon reload + + - name: Deploy handler units + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy-handler@.service.j2 + dest: /etc/systemd/system/galaxy-handler@.service + notify: daemon reload + + - name: Deploy workflow units + template: + owner: root + group: root + mode: 0644 + src: systemd/galaxy-workflow-scheduler@.service.j2 + dest: /etc/systemd/system/galaxy-workflow-scheduler@.service + when: galaxy_systemd.processes.workflow > 0 + notify: daemon reload + + # TODO: In case of changes? Not sure this is necessary. + #- name: just force systemd to reread configs (2.4 and above) + #systemd: + #daemon_reload: yes + + - name: Enable zergpool and ensure it is running + systemd: + name: galaxy-zergpool.service + enabled: yes + state: started + + - name: Enable relevant zerglings + systemd: + name: "galaxy-zergling@{{ item }}.service" + enabled: yes + state: started + loop: "{{ range(0, galaxy_systemd.processes.zerglings) | list }}" + + - name: Enable relevant handlers + systemd: + name: "galaxy-handler@{{ item }}.service" + enabled: yes + state: started + loop: "{{ range(0, galaxy_systemd.processes.handlers) | list }}" + + - name: Enable relevant workflows + systemd: + name: "galaxy-workflow-scheduler@{{ item }}.service" + enabled: yes + state: started + loop: "{{ range(0, galaxy_systemd.processes.workflow) | list }}" + when: galaxy_systemd_workflow_schedulers > 0 + + remote_user: "{{ galaxy_remote_users.root | default(__galaxy_remote_user) }}" + become: "{{ true if galaxy_become_users.root is defined else __galaxy_become }}" + become_user: "{{ galaxy_become_users.root | default(__galaxy_become_user) }}" diff --git a/templates/systemd/galaxy-handler@.service.j2 b/templates/systemd/galaxy-handler@.service.j2 new file mode 100644 index 0000000..3bcbef6 --- /dev/null +++ b/templates/systemd/galaxy-handler@.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Galaxy Handlers +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=30 +ExecStart={{ galaxy_venv_dir }}/bin/python ./scripts/galaxy-main -c {{ galaxy_config_dir }}/{{ galaxy_config_file_basename }} --server-name=handler_{{ galaxy_instance_codename }}_%I +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin DOCUTILSCONFIG='' PYTHONPATH={{ galaxy_dynamic_job_rules_dir }} {{ galaxy_systemd.env }} +MemoryLimit={{ galaxy_systemd.memory_limit.handler }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/galaxy-reports.service.j2 b/templates/systemd/galaxy-reports.service.j2 new file mode 100644 index 0000000..4729aa6 --- /dev/null +++ b/templates/systemd/galaxy-reports.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Galaxy Reports +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=10 +ExecStart={{ galaxy_venv_dir }}/bin/uwsgi {{ '--yaml' if galaxy_config_style in ('yaml', 'yml') else '--ini' }} {{ galaxy_reports_path }} --stats 127.0.0.1:4030 +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin DOCUTILSCONFIG= PYTHONPATH={{ galaxy_dynamic_job_rules_dir }} {{ galaxy_systemd.env }} +MemoryLimit={{ galaxy_systemd.memory_limit.reports }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/galaxy-workflow-scheduler@.service.j2 b/templates/systemd/galaxy-workflow-scheduler@.service.j2 new file mode 100644 index 0000000..905311a --- /dev/null +++ b/templates/systemd/galaxy-workflow-scheduler@.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Galaxy Workflow Scheduler +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=30 +ExecStart={{ galaxy_venv_dir }}/bin/python ./scripts/galaxy-main -c {{ galaxy_config_dir }}/{{ galaxy_config_file_basename }} --server-name=workflow_scheduler_{{ galaxy_instance_codename }}_%I +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin DOCUTILSCONFIG='' PYTHONPATH={{ galaxy_dynamic_job_rules_dir }} {{ galaxy_systemd.env }} +MemoryLimit={{ galaxy_systemd.memory_limit.workflow }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/galaxy-zergling@.service.j2 b/templates/systemd/galaxy-zergling@.service.j2 new file mode 100644 index 0000000..f6a6c2e --- /dev/null +++ b/templates/systemd/galaxy-zergling@.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Galaxy Zerglings +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=10 +ExecStart={{ galaxy_venv_dir }}/bin/uwsgi {{ '--yaml' if galaxy_systemd_zergling_uwsgi_config_style in ('yaml', 'yml') else '--ini' }} {{ galaxy_systemd_zergling_uwsgi_config_file }} --stats 127.0.0.1:401%I {% if galaxy_zergpool %}--zerg {{ galaxy_zergpool_listen_path }}{% else %}--socket 127.0.0.1:400%I{% endif %} + +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin DOCUTILSCONFIG='' PYTHONPATH={{ galaxy_dynamic_job_rules_dir }} {{ galaxy_systemd.env }} +MemoryLimit={{ galaxy_systemd.memory_limit.zergling }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/galaxy-zergpool.service.j2 b/templates/systemd/galaxy-zergpool.service.j2 new file mode 100644 index 0000000..f5bf3f6 --- /dev/null +++ b/templates/systemd/galaxy-zergpool.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Galaxy Zergpool +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=10 +ExecStart={{ galaxy_venv_dir }}/bin/uwsgi --master --zerg-pool {{ galaxy_zergpool_listen_path }}:{{ galaxy_zergpool_listen_addr }} +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin +MemoryLimit={{ galaxy_systemd.memory_limit.zergpool }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/galaxy.service.j2 b/templates/systemd/galaxy.service.j2 new file mode 100644 index 0000000..1112d39 --- /dev/null +++ b/templates/systemd/galaxy.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Galaxy +After=network.target +After=time-sync.target + +[Service] +UMask=022 +Type=simple +User={{ galaxy_user.name }} +Group={{ __galaxy_user_group }} +WorkingDirectory={{ galaxy_server_dir }} +TimeoutStartSec=10 +ExecStart={{ galaxy_venv_dir }}/bin/uwsgi {{ '--yaml' if galaxy_config_style in ('yaml', 'yml') else '--ini' }} {{ galaxy_config_file }} {% if galaxy_systemd.stats %}--stats 127.0.0.1:{{ galaxy_systemd.stats }}{% endif %} +Environment=HOME={{ galaxy_root }} VIRTUAL_ENV={{ galaxy_venv_dir }} PATH={{ galaxy_venv_dir }}/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin DOCUTILSCONFIG= PYTHONPATH={{ galaxy_dynamic_job_rules_dir }} {{ galaxy_systemd.env }} +MemoryLimit={{ galaxy_systemd.memory_limit.mule }}G +Restart=always + +MemoryAccounting=yes +CPUAccounting=yes +BlockIOAccounting=yes + +[Install] +WantedBy=multi-user.target diff --git a/templates/systemd/resource_controls.conf.j2 b/templates/systemd/resource_controls.conf.j2 new file mode 100644 index 0000000..0bbaff9 --- /dev/null +++ b/templates/systemd/resource_controls.conf.j2 @@ -0,0 +1,31 @@ +;; +;; This file is managed by Ansible. ALL CHANGES WILL BE OVERWRITTEN. +;; +{% if item.real_memory_limit is defined %} +{% set memlim = item.real_memory_limit %} +{% elif item.real_memory_limit_percent is defined and item.real_memory_limit_percent > 1.0 %} +{% set memlim = ((item.real_memory_limit_percent / 100.0) * ansible_memory_mb.real.total) | int ~ "M" %} +{% elif item.real_memory_limit_percent is defined %} +{% set memlim = (item.real_memory_limit_percent * ansible_memory_mb.real.total) | int ~ "M" %} +{% else %} +{% set memlim = none %} +{% endif -%} + +{%- set memlim_mb = memlim | default(ansible_memory_mb.real.total * 1024 * 1024) | human_to_bytes / 1024 // 1024 %} +{% if item.swap_limit is defined %} +{% set memswlim = (memlim_mb + (item.swap_limit | human_to_bytes / 1024 // 1024)) ~ "M" %} +{% elif item.swap_limit_percent is defined and item.swap_limit_percent > 1.0 %} +{% set memswlim = (memlim_mb + (item.swap_limit_percent / 100.0) * ansible_memory_mb.swap.total) | int ~ "M" %} +{% elif item.swap_limit_percent is defined %} +{% set memswlim = (memlim_mb + item.swap_limit_percent * ansible_memory_mb.swap.total) | int ~ "M" %} +{% else %} +{% set memswlim = none %} +{% endif -%} + +[{{ item.unit_type | default("service") | title }}] +{% if memlim is not none %} +MemoryLimit={{ memlim }} +{% endif %} +{% if memswlim is not none %} +ExecStartPost=/bin/bash -c "echo {{ memswlim }} > /sys/fs/cgroup/memory/system.slice/{{ item.name }}.{{ item.unit_type | default('service') | lower }}/memory.memsw.limit_in_bytes" +{% endif %}