Merge branch 'master' into main

.travis.yml

@@ -22,7 +22,7 @@ install:
   - pip install --upgrade pip
 
   # Install Ansible.
-  - pip install ansible==2.4
+  - pip install ansible==2.5
 
   # Add ansible.cfg to pick up roles path.
   - printf '[defaults]\nroles_path = ../' > ansible.cfg

README.md

@@ -48,12 +48,41 @@ with the `virtualenv` executable and corresponding `pip_virtualenv_command` vari
 Role Variables
 --------------
 
+Many variables control where specific files are placed and where Galaxy writes data. In order to simplify configuration,
+you may select a *layout* with the `galaxy_layout` variable. Which layout you choose affects the required variables.
+
 ### Required variables ###
 
+If using any layout other than `root-dir`:
+
 - `galaxy_server_dir`: Filesystem path where the Galaxy server code will be installed (cloned).
 
+If using `root-dir`:
+
+- `galaxy_root`: Filesystem path of the root of a Galaxy deployment, the Galaxy server code will be installed in to a
+  subdirectory of this directory.
+
 ### Optional variables ###
 
+Layout control:
+
+- `galaxy_layout`: available layouts can be found in the [vars/][vars] subdirectory and possible values include:
+  - `root-dir`: Everything is laid out in subdirectories underneath a single root directory.
+  - `opt`: An [FHS][fhs]-conforming layout across multiple directories such as `/opt`, `/etc/opt`, etc.
+  - `legacy-improved`: Everything underneath the Galaxy server directory, as with `run.sh`.
+  - `legacy`: The default layout prior to the existence of `galaxy_layout` and currently the default so as not to break
+    existing usage of this role.
+  - `custom`: Reasonable defaults for custom layouts, requires setting a few variables as described in
+    [vars/layout-custom.yml][custom]
+
+Either the `root-dir` or `opt` layout is recommended for new Galaxy deployments.
+
+Options below that control individual file or subdirectory placement can still override defaults set by the layout.
+
+[vars]: vars/
+[custom]: vars/layout-custom.yml
+[fhs]: http://www.pathname.com/fhs/
+
 New options for Galaxy 18.01 and later:
 
 - `galaxy_config_style` (default: `ini-paste`): The type of Galaxy configuration file to write, `ini-paste` for the
@@ -125,7 +154,6 @@ will be placed:
   reached" on the 502 page.
 - `galaxy_errordocs_prefix` (default: `/error`): Web-side path to the error document root.
 
-
 Dependencies
 ------------
 
@@ -166,11 +194,8 @@ and start two [job handler mules][deployment-options].
 - hosts: galaxyservers
   vars:
     galaxy_config_style: yaml
-    galaxy_server_dir: /opt/galaxy/server
-    galaxy_config_dir: /opt/galaxy/config
-    galaxy_mutable_config_dir: /var/opt/galaxy/config
-    galaxy_mutable_data_dir: /var/opt/galaxy/data
-    galaxy_commit_id: release_18.05
+    galaxy_layout: opt
+    galaxy_commit_id: release_18.09
     postgresql_objects_users:
       - name: galaxy
         password: null
@@ -198,8 +223,9 @@ and start two [job handler mules][deployment-options].
           - unix_signal:15 gracefully_kill_them_all
         py-call-osafterfork: true
         enable-threads: true
-        mule: lib/galaxy/main.py
-        mule: lib/galaxy/main.py
+        mule:
+          - lib/galaxy/main.py
+          - lib/galaxy/main.py
         farm: job-handlers:1,2
       galaxy:
         database_connection: "postgresql:///galaxy?host=/var/run/postgresql"
@@ -235,7 +261,8 @@ and start two [job handler mules][deployment-options].
   roles:
     # Install with:
     #   % ansible-galaxy install galaxyproject.postgresql
-    - galaxyproject.postgresql
+    - role: galaxyproject.postgresql
+      become: yes
     # Install with:
     #   % ansible-galaxy install natefoo.postgresql_objects
     - role: natefoo.postgresql_objects
@@ -265,8 +292,8 @@ Author Information
 
 This role was written and contributed to by the following people:
 
-[Enis Afgan](https://github.com/afgane)  
-[Dannon Baker](https://github.com/dannon)  
-[Simon Belluzzo](https://github.com/simonalpha)  
-[John Chilton](https://github.com/jmchilton)  
-[Nate Coraor](https://github.com/natefoo)  
+- [Enis Afgan](https://github.com/afgane)
+- [Dannon Baker](https://github.com/dannon)
+- [Simon Belluzzo](https://github.com/simonalpha)
+- [John Chilton](https://github.com/jmchilton)
+- [Nate Coraor](https://github.com/natefoo)

defaults/main.yml

@@ -5,14 +5,17 @@
 # options on different hosts or server directories.
 galaxy_manage_clone: yes
 galaxy_force_checkout: no
-# Download the Galaxy instead cloning, for throw away boxes (e.g. Docker)
+# Download Galaxy instead of cloning, for throw away boxes (e.g. Docker)
 galaxy_manage_download: no
 galaxy_manage_static_setup: yes
 galaxy_manage_mutable_setup: yes
 galaxy_manage_database: yes
 galaxy_fetch_dependencies: yes
 galaxy_manage_errordocs: no
 
+# Directory layout for deployment (legacy is the default)
+#galaxy_layout: legacy
+
 # Where to clone Galaxy from. Backwards compatibility with galaxy_git_repo
 galaxy_repo: "{{ galaxy_git_repo | default('https://github.com/galaxyproject/galaxy.git') }}"
 
@@ -22,21 +25,15 @@ galaxy_repo: "{{ galaxy_git_repo | default('https://github.com/galaxyproject/gal
 galaxy_commit_id: "{{ galaxy_changeset_id | default('master') }}"
 
 # Download URL (if used)
-galaxy_download_url: "{{ galaxy_repo }}/get/{{ galaxy_commit_id }}.tar.gz"
-
-# This will automatically be used by run.sh
-galaxy_venv_dir: "{{ galaxy_server_dir }}/.venv"
+galaxy_download_url: "{{ galaxy_repo | replace('.git', '') }}/archive/{{ galaxy_commit_id }}.tar.gz"
 
 # TODO: Implement something like this for starter/restarter
 #galaxy_server_type: paste_singleprocess
 
-# By default, everything lives in the Galaxy clone
-galaxy_config_dir: "{{ galaxy_server_dir }}"
-galaxy_mutable_config_dir: "{{ galaxy_server_dir }}"
-galaxy_tool_dependency_dir: "{{ galaxy_mutable_data_dir }}/dependencies"
-galaxy_mutable_data_dir: "{{ galaxy_server_dir }}/database"
-galaxy_shed_tools_dir: "{{ galaxy_server_dir }}/../shed_tools"
+# The shed tool conf path is needed separate from the value of tool_conf for instantiation
 galaxy_shed_tool_conf_file: "{{ galaxy_mutable_config_dir }}/shed_tool_conf.xml"
+
+# Avoid hardcoding paths that could change in tasks
 galaxy_requirements_file: "{{ galaxy_server_dir }}/lib/galaxy/dependencies/pinned-requirements.txt"
 
 # Galaxy config file can either be 'ini-paste' (old style) or 'yaml'
@@ -50,31 +47,41 @@ galaxy_app_config_section: "{{ 'galaxy' if galaxy_config_style in ('yaml', 'yml'
 # configs if galaxy_config_dir != galaxy_server_dir
 galaxy_paste_app_factory: "galaxy.web.buildapp:app_factory"
 galaxy_app_config_default:
+  # Defaults for sample configs that require a value to be set
   builds_file_path: "{{ galaxy_server_dir }}/tool-data/shared/ucsc/builds.txt.sample"
   data_manager_config_file: "{{ galaxy_server_dir }}/config/data_manager_conf.xml.sample"
   datatypes_config_file: "{{ galaxy_server_dir }}/config/datatypes_conf.xml.sample"
   external_service_type_config_file: "{{ galaxy_server_dir }}/config/external_service_types_conf.xml.sample"
-  integrated_tool_panel_config: "{{ galaxy_mutable_config_dir }}/integrated_tool_panel.xml"
-  migrated_tools_config: "{{ galaxy_mutable_config_dir }}/migrated_tools_conf.xml"
   openid_config_file: "{{ galaxy_server_dir }}/config/openid_conf.xml.sample"
-  shed_data_manager_config_file: "{{ galaxy_mutable_config_dir }}/shed_data_manager_conf.xml"
-  shed_tool_data_table_config: "{{ galaxy_mutable_config_dir }}/shed_tool_data_table_conf.xml"
-  tool_config_file: "{{ galaxy_server_dir }}/config/tool_conf.xml.sample,{{ galaxy_shed_tool_conf_file }}"
+  ucsc_build_sites: "{{ galaxy_server_dir }}/tool-data/shared/ucsc/ucsc_build_sites.txt.sample"
   tool_data_table_config_path: "{{ galaxy_server_dir }}/config/tool_data_table_conf.xml.sample"
   tool_sheds_config_file: "{{ galaxy_server_dir }}/config/tool_sheds_conf.xml.sample"
-  ucsc_build_sites: "{{ galaxy_server_dir }}/tool-data/shared/ucsc/ucsc_build_sites.txt.sample"
-  visualization_plugins_directory: "config/plugins/visualizations"
 
-  file_path: "{{ galaxy_mutable_data_dir }}/datasets"
+  # Various cache and temporary directories
+  template_cache_path: "{{ galaxy_cache_dir }}/template_cache"
+  whoosh_index_dir: "{{ galaxy_cache_dir }}/whoosh_cache"
+  object_store_cache_path: "{{ galaxy_cache_dir }}/object_store_cache"
+  openid_consumer_cache_path: "{{ galaxy_cache_dir }}/openid_consumer_cache"
+  citation_cache_data_dir: "{{ galaxy_cache_dir }}/citations/data"
+  citation_cache_lock_dir: "{{ galaxy_cache_dir }}/citations/locks"
   new_file_path: "{{ galaxy_mutable_data_dir }}/tmp"
-  job_working_directory: "{{ galaxy_mutable_data_dir }}/jobs"
+
+  # Mutable config files
+  integrated_tool_panel_config: "{{ galaxy_mutable_config_dir }}/integrated_tool_panel.xml"
+  migrated_tools_config: "{{ galaxy_mutable_config_dir }}/migrated_tools_conf.xml"
+  shed_data_manager_config_file: "{{ galaxy_mutable_config_dir }}/shed_data_manager_conf.xml"
+  shed_tool_data_table_config: "{{ galaxy_mutable_config_dir }}/shed_tool_data_table_conf.xml"
+
+  # Directories that would often be stored outside the mutable data dir, these values are either from the layout or set by the user
+  file_path: "{{ galaxy_file_path }}"
+  job_working_directory: "{{ galaxy_job_working_directory }}"
   tool_dependency_dir: "{{ galaxy_tool_dependency_dir }}"
-  template_cache_path: "{{ galaxy_mutable_data_dir }}/template_cache"
-  whoosh_index_dir: "{{ galaxy_mutable_data_dir }}/whoosh_cache"
-  object_store_cache_path: "{{ galaxy_mutable_data_dir }}/object_store_cache"
-  openid_consumer_cache_path: "{{ galaxy_mutable_data_dir }}/openid_consumer_cache"
-  citation_cache_data_dir: "{{ galaxy_mutable_data_dir }}/citations/data"
-  citation_cache_lock_dir: "{{ galaxy_mutable_data_dir }}/citations/locks"
+
+  # Static config files
+  tool_config_file: "{{ galaxy_server_dir }}/config/tool_conf.xml.sample,{{ galaxy_shed_tool_conf_file }}"
+
+  # Everything else
+  visualization_plugins_directory: "config/plugins/visualizations"
 
 # Need to set galaxy_config_default[galaxy_app_config_section] dynamically but Ansible/Jinja2 does not make this easy
 galaxy_config_default: "{{ {} | combine({galaxy_app_config_section: galaxy_app_config_default}) }}"

meta/main.yml

@@ -4,7 +4,7 @@ galaxy_info:
   description: Install and manage a Galaxy (http://galaxyproject.org/) server.
   company: The Galaxy Project
   license: AFL v3.0
-  min_ansible_version: 2.4
+  min_ansible_version: 2.5
   platforms:
   - name: EL
     versions:

tasks/database.yml

@@ -22,6 +22,12 @@
     chdir: "{{ galaxy_server_dir }}"
   when: "'migrate.exceptions.DatabaseNotControlledError' in current_db_version.stderr"
 
+- name: Report current and max Galaxy database
+  debug:
+    msg: "Current database version is {{ current_db_version.stdout }} and the maxiumum version is {{ max_db_version.stdout }}."
+  changed_when: True
+  when: current_db_version.stdout != max_db_version.stdout and 'migrate.exceptions.DatabaseNotControlledError' not in current_db_version.stderr
+
 - name: Upgrade Galaxy DB
   command: "{{ galaxy_venv_dir }}/bin/python {{ galaxy_server_dir }}/scripts/manage_db.py -c {{ galaxy_config_file }} upgrade"
   args:

tasks/download.yml

@@ -1,11 +1,34 @@
+---
+# Manage Galaxy download
+
 - name: Check for Galaxy download receipt
   stat:
     path: "{{ galaxy_server_dir }}/{{ galaxy_commit_id }}_receipt"
   register: download_receipt
 
-- name: Replace current version of Galaxy
-  shell: "rm -rf '{{ galaxy_server_dir }}' && mkdir '{{ galaxy_server_dir }}' && wget -q -O - {{ galaxy_download_url }} | tar xzf - --strip-components=1 -C {{ galaxy_server_dir }} && touch '{{ galaxy_server_dir }}/{{ galaxy_commit_id }}_receipt'"
+- name: Create Galaxy server directory
+  file:
+    path: "{{ galaxy_server_dir }}"
+    state: directory
+
+- name: Install current version of Galaxy
+  unarchive:
+    src: "{{ galaxy_download_url }}"
+    dest: "{{ galaxy_server_dir }}"
+    extra_opts: --strip-components=1
+    remote_src: yes
+  when: not download_receipt.stat.exists
+
+- name: Create Galaxy download receipt
+  file:
+    path: "{{ galaxy_server_dir }}/{{ galaxy_commit_id }}_receipt"
+    state: touch
   when: not download_receipt.stat.exists
 
 - name: Include virtualenv setup tasks
   include_tasks: virtualenv.yml
+
+- name: Remove orphaned .pyc files and compile bytecode
+  script: makepyc.py {{ galaxy_server_dir }}/lib
+  environment:
+    PATH: "{{ galaxy_venv_dir }}/bin"

tasks/layout.yml

@@ -0,0 +1,40 @@
+---
+# Configure any unset variables using layout defaults
+
+- name: Include layout vars
+  include_vars: "layout-{{ galaxy_layout | default('legacy') }}.yml"
+
+- name: Set any unset variables from layout defaults
+  set_fact:
+    "{{ item }}": "{{ lookup('vars', '__' ~ item) }}"
+  when: item not in vars
+  with_items:
+    - galaxy_venv_dir
+    - galaxy_server_dir
+    - galaxy_config_dir
+    - galaxy_mutable_config_dir
+    - galaxy_mutable_data_dir
+    - galaxy_shed_tools_dir
+    - galaxy_cache_dir
+
+- name: Check that any explicitly set Galaxy config options match the values of explicitly set variables
+  assert:
+    that:
+      - lookup('vars', 'galaxy_' ~ item) == galaxy_config[galaxy_app_config_section][item]
+    msg: "Value of '{{ 'galaxy_' ~ item }}' does not match value of '{{ item }}' in galaxy_config: {{ lookup('vars', 'galaxy_' ~ item) }} != {{ galaxy_config[galaxy_app_config_section][item] }}"
+    # TODO: requires Ansible 2.7
+    #success_msg: "Value of '{{ 'galaxy_' ~ item }}' matches galaxy_config value of '{{ item }}' in galaxy_config"
+  when: "'galaxy_' ~ item in vars and item in ((galaxy_config | default({}))[galaxy_app_config_section] | default({}))"
+  with_items:
+    - tool_dependency_dir
+    - file_path
+    - job_working_directory
+
+- name: Set any unset variables corresponding to Galaxy config options from galaxy_config or layout defaults
+  set_fact:
+    "{{ 'galaxy_' ~ item }}": "{{ ((galaxy_config | default({}))[galaxy_app_config_section] | default({}))[item] | default(lookup('vars', '__galaxy_' ~ item)) }}"
+  when: "'galaxy_' ~ item not in vars"
+  with_items:
+    - tool_dependency_dir
+    - file_path
+    - job_working_directory

tasks/main.yml

@@ -2,12 +2,16 @@
 # main tasks file for galaxyproject.galaxy
 
 # Deploy a Galaxy server
-#   https://wiki.galaxyproject.org/
+#   https://docs.galaxyproject.org/
 
 - fail:
     msg: "Variables galaxy_manage_clone and galaxy_manage_download cannot both be true."
   when: galaxy_manage_clone and galaxy_manage_download
 
+- name: Import layout variable tasks
+  import_tasks: layout.yml
+  tags: always
+
 - name: Include clone tasks
   include_tasks: clone.yml
   when: galaxy_manage_clone

tasks/virtualenv.yml

@@ -9,6 +9,7 @@
     name: pip
     virtualenv: "{{ galaxy_venv_dir }}"
     virtualenv_command: "{{ pip_virtualenv_command | default('virtualenv') }}"
+    virtualenv_python: python2.7
   environment:
     VIRTUAL_ENV: "{{ galaxy_venv_dir }}"
 

vars/layout-custom.yml

@@ -0,0 +1,17 @@
+---
+# Reasonable defaults for custom layouts. To use, you must define:
+#
+# - galaxy_server_dir
+# - galaxy_venv_dir
+# - galaxy_config_dir
+# - galaxy_mutable_data_dir
+
+# Layout defaults
+__galaxy_mutable_config_dir: "{{ galaxy_mutable_data_dir }}/config"
+__galaxy_cache_dir: "{{ galaxy_mutable_data_dir }}/cache"
+__galaxy_shed_tools_dir: "{{ galaxy_mutable_data_dir }}/shed_tools"
+
+# These correspond to galaxy config options
+__galaxy_tool_dependency_dir: "{{ galaxy_mutable_data_dir }}/dependencies"
+__galaxy_file_path: "{{ galaxy_mutable_data_dir }}/datasets"
+__galaxy_job_working_directory: "{{ galaxy_mutable_data_dir }}/jobs"

vars/layout-legacy-improved.yml

@@ -0,0 +1,14 @@
+---
+
+# Layout defaults
+__galaxy_venv_dir: "{{ galaxy_server_dir }}/.venv"
+__galaxy_config_dir: "{{ galaxy_server_dir }}/config"
+__galaxy_mutable_data_dir: "{{ galaxy_server_dir }}/database"
+__galaxy_mutable_config_dir: "{{ galaxy_server_dir }}/config"
+__galaxy_cache_dir: "{{ galaxy_mutable_data_dir }}"
+__galaxy_shed_tools_dir: "{{ galaxy_mutable_data_dir }}/shed_tools"
+
+# These correspond to galaxy config options
+__galaxy_tool_dependency_dir: "{{ galaxy_mutable_data_dir }}/dependencies"
+__galaxy_file_path: "{{ galaxy_mutable_data_dir }}/datasets"
+__galaxy_job_working_directory: "{{ galaxy_mutable_data_dir }}/jobs"

vars/layout-legacy.yml

@@ -0,0 +1,14 @@
+---
+
+# Layout defaults
+__galaxy_venv_dir: "{{ galaxy_server_dir }}/.venv"
+__galaxy_config_dir: "{{ galaxy_server_dir }}"
+__galaxy_mutable_data_dir: "{{ galaxy_server_dir }}/database"
+__galaxy_mutable_config_dir: "{{ galaxy_server_dir }}"
+__galaxy_cache_dir: "{{ galaxy_mutable_data_dir }}"
+__galaxy_shed_tools_dir: "{{ galaxy_server_dir }}/../shed_tools"
+
+# These correspond to galaxy config options
+__galaxy_tool_dependency_dir: "{{ galaxy_mutable_data_dir }}/dependencies"
+__galaxy_file_path: "{{ galaxy_mutable_data_dir }}/datasets"
+__galaxy_job_working_directory: "{{ galaxy_mutable_data_dir }}/jobs"