diff --git a/BrainPortal/app/controllers/sessions_controller.rb b/BrainPortal/app/controllers/sessions_controller.rb index 989a4ff8e..faf32c0d9 100644 --- a/BrainPortal/app/controllers/sessions_controller.rb +++ b/BrainPortal/app/controllers/sessions_controller.rb @@ -42,7 +42,7 @@ def new #:nodoc: # HEAD requests render nothing req_method = request.method.to_s.upcase if req_method == 'HEAD' - render :plain => "", :status => :ok + head :ok return end diff --git a/BrainPortal/app/models/cluster_task.rb b/BrainPortal/app/models/cluster_task.rb index d23df027d..15bbfe447 100644 --- a/BrainPortal/app/models/cluster_task.rb +++ b/BrainPortal/app/models/cluster_task.rb @@ -2373,7 +2373,7 @@ def singularity_commands(command_script) # must be on a device different from the one for the work directory. capture_basenames = ext3capture_basenames.map { |basename,_| basename } - # (4) More -B (bind mounts) for all the relevant local data providers. + # (4a) More -B (bind mounts) for all the relevant local data providers. # This will be a string "-B path1 -B path2 -B path3" etc. # In the case of read-only input files, ro option is added esc_local_dp_mountpoints = local_dp_storage_paths.inject("") do |sing_opts,path| @@ -2382,6 +2382,12 @@ def singularity_commands(command_script) sing_opts end + # (4b) Add tool config bindmounts, specified in 'overlay' + esc_local_bindmounts = bindmount_paths.inject("") do |sing_opts,(from_path,cont_path)| + sing_opts += " -B #{from_path.bash_escape}:#{cont_path.bash_escape}" + sing_opts + end + # (5) Overlays defined in the ToolConfig # Some of them might be patterns (e.g. /a/b/data*.squashfs) that need to # be resolved locally. @@ -2497,7 +2503,8 @@ def singularity_commands(command_script) # a) at its original cluster full path # b) at /DP_Cache (used only when shortening workdir) # 3) we mount the root of the gridshare area (for all tasks) -# 4) we mount each (if any) of the root directories for local data providers +# 4a) we mount each (if any) of the root directories for local data providers +# 4b) we mount each (if any) of the bindmounts configured in the ToolConfig # 5) we mount (if any) other fixed file system overlays # 6) we mount (if any) capture ext3 filesystems # 7) with -H we set the task's work directory as the singularity $HOME directory @@ -2508,6 +2515,7 @@ def singularity_commands(command_script) -B #{cache_dir.bash_escape}:/DP_Cache \\ -B #{gridshare_dir.bash_escape} \\ #{esc_local_dp_mountpoints} \\ + #{esc_local_bindmounts} \\ #{overlay_mounts} \\ -B #{task_workdir.bash_escape}:#{effect_workdir.bash_escape} \\ #{esc_capture_mounts} \\ @@ -2566,6 +2574,12 @@ def ext3capture_basenames self.tool_config.ext3capture_basenames end + # Just invokes the same method on the task's ToolConfig. + # Returns an array of pairs, e.g. [ [ src, dest ], [ src, dest ] ] + def bindmount_paths + self.tool_config.bindmount_paths + end + # This method creates an empty +filename+ with +size+ bytes # (where size is specified like what the unix 'truncate' command accepts) # and then formats it with a ext3 filesystem. If the filename already exists, diff --git a/BrainPortal/app/models/tool_config.rb b/BrainPortal/app/models/tool_config.rb index 7a2574d55..cbe630459 100644 --- a/BrainPortal/app/models/tool_config.rb +++ b/BrainPortal/app/models/tool_config.rb @@ -384,8 +384,10 @@ def cbrain_task_class # dp:1234 # # CBRAIN db registered file # userfile:1234 - # # A ext3 capture filesystem, will NOT be returned here as an overlay + # # A ext3 capture filesystem, will NOT be returned here as an overlay (see method ext3capture_basenames() instead) # ext3capture:basename=12G + # # A bind mount, will NOT be returned here as an overlay (see method bindmount_paths() instead) + # bindmount:/local/basename:/container/basename def singularity_overlays_full_paths specs = parsed_overlay_specs specs.map do |knd, id_or_name| @@ -410,6 +412,8 @@ def singularity_overlays_full_paths { userfile.cache_full_path() => "registered userfile" } when 'ext3capture' [] # handled separately + when 'bindmount' + [] # handled separately else cb_error "Invalid '#{knd}:#{id_or_name}' overlay." end @@ -417,7 +421,7 @@ def singularity_overlays_full_paths end # Returns an array of the data providers that are - # specified in the attribute singularity_overlays_specs, + # specified in the attribute +singularity_overlays_specs+, # ignoring all other overlay specs for normal files. def data_providers_with_overlays return @_data_providers_with_overlays_ if @_data_providers_with_overlays_ @@ -428,6 +432,39 @@ def data_providers_with_overlays end.compact end + # Returns an array of pairs extracted from the attribute + # +singularity_overlays_specs+ , ignoring all other overlay + # specs for normal files. + def bindmount_paths + specs = parsed_overlay_specs.presence || [] + + # Pairs of paths obtained from the ToolConfig's "overlay" configuration. + tc_paths = specs + .map { |pair| pair[1] if pair[0] == 'bindmount' } + .compact + .map { |frompath_contpath| frompath_contpath.split(":",2) } + + # One additional bindmount path from the plugins directory where the tool + # comes from. Available ONLY if tool is configured with + # a boutiques descriptor. The following lines of code make + # a bunch of checks and as soon as a check fails, we just + # return with the array tc_paths computed above. + descriptor = self.boutiques_descriptor rescue nil + return tc_paths if descriptor.blank? + container_mountpoint = descriptor.custom["cbrain:plugins-container-bindmount"] + return tc_paths if container_mountpoint.blank? + desc_file = descriptor.from_file # "/path/to/RailsApp/cbrain-plugins/installed_plugins/boutiques_descriptors/toolname.json" + return tc_paths if desc_file.blank? + real_path = File.realpath(desc_file) # "/path/to/RailsApp/cbrain-plugins/plugin-name/boutiques_descriptors/toolname.json" + parent1 = Pathname.new(real_path).parent # "/path/to/RailsApp/cbrain-plugins/plugin-name/boutiques_descriptors" + return tc_paths if parent1.basename.to_s != "boutiques_descriptors" # check plugins convention + plugin_path = parent1.parent # "/path/to/RailsApp/cbrain-plugins/plugin-name" + plugin_containerized_dir = plugin_path + "container_mnt" # special plugins folder to mount + return tc_paths if ! File.directory?(plugin_containerized_dir.to_s) + tc_paths << [ plugin_containerized_dir.to_s, container_mountpoint + ":ro" ] + return tc_paths + end + # Returns pairs [ [ basename, size], ...] as in [ [ 'work', '28g' ] def ext3capture_basenames specs = parsed_overlay_specs @@ -475,7 +512,8 @@ def validate_container_rules #:nodoc: return errors.empty? end - # breaks down overlay spec onto a list of overlays + # Breaks down the singularity_overlays_specs attribute, and returns a list of pairs [ type, value ] e.g. + # [ [ 'userfile', '1234' ], [ 'file', '/hello/bye/data.sqs' ], [ 'bindmount', '/some/data/dir:/mount/this/here' ] ] def parsed_overlay_specs specs = self.singularity_overlays_specs return [] if specs.blank? @@ -486,8 +524,9 @@ def parsed_overlay_specs end # Verifies that the admin has entered a set of - # overlay specifications properly. One or several of: + # overlay and bindmount specifications properly. One or several of: # + # #### the overlay rules # file:/full/path/to/something.squashfs # file:/full/path/to/pattern*/data?.squashfs # userfile:333 @@ -497,6 +536,8 @@ def parsed_overlay_specs # ext3capture:work=30G # ext3capture:tool_1.1.2=15M # + # bindmount:/bourreau/path/to:/container/path/to # bindmount rules ( additionally introduced ) + # def validate_overlays_specs #:nodoc: specs = parsed_overlay_specs @@ -542,6 +583,10 @@ def validate_overlays_specs #:nodoc: self.errors.add(:singularity_overlays_specs, "contains invalid ext3capture specification (must be like ext3capture:basename=1g or 2m etc)") end + when 'bindmount' # this is for binding to container rather than overlays + if id_or_name !~ /\A\/\w[\w\.\-\/]+:\/\w[\w\.\-\/]+(:ro)?\z/ + self.errors.add(:singularity_overlays_specs, "contains invalid bindmount specification (must be like 'bindmount:/path/in/bourreau:/path/in/container' with or without a final ':ro') and free from special characters") + end else # Other errors self.errors.add(:singularity_overlays_specs, "contains invalid specification '#{kind}:#{id_or_name}'") diff --git a/BrainPortal/app/views/bourreaux/_bourreaux_display.html.erb b/BrainPortal/app/views/bourreaux/_bourreaux_display.html.erb index 34a4a3315..8d1724896 100644 --- a/BrainPortal/app/views/bourreaux/_bourreaux_display.html.erb +++ b/BrainPortal/app/views/bourreaux/_bourreaux_display.html.erb @@ -407,7 +407,7 @@ Stopping an Execution Server will also stop the Task Workers and Activity Workers running on it. Make sure they are not actively <%= html_colorize("processing","orange") %> something.

- <%= external_submit_button "2. Stop Bourreaux", "bourreau_form", + <%= external_submit_button "2. Stop Execution Server", "bourreau_form", :class => 'button', :name => 'operation', :value => 'stop_bourreaux', @@ -421,7 +421,7 @@ :class => 'button', :name => 'operation', :value => 'stop_tunnels', - :data => { :confirm => "Make sure Bourreaux, Task Workers and Activity workers are all stopped!" } + :data => { :confirm => "Make sure Execution Servers, Task Workers and Activity workers are all stopped!" } %> <% end %> diff --git a/BrainPortal/app/views/tool_configs/_form_fields.html.erb b/BrainPortal/app/views/tool_configs/_form_fields.html.erb index e3cd5b392..4125aace1 100644 --- a/BrainPortal/app/views/tool_configs/_form_fields.html.erb +++ b/BrainPortal/app/views/tool_configs/_form_fields.html.erb @@ -222,17 +222,30 @@ <% t.edit_cell(:singularity_overlays_specs, :header => "Singularity Overlays", :show_width => 2, :content => full_description(@tool_config.singularity_overlays_specs)) do |f| %> <%= f.text_area :singularity_overlays_specs, :rows => 6, :cols => 120 %>
- This field can contain one or several specifications for data overlays + This field can contain one or several specifications for data overlays and bindmounts to be included when the task is started with Singularity. - A specification can be either - a full path (e.g. file:/a/b/data.squashfs), - a path with a pattern (e.g. file:/a/b/data*.squashfs), - a registered file identified by ID (e.g. userfile:123), - a SquashFS Data Provider identified by its ID or name (e.g. dp:123, dp:DpNameHere) - or an ext3 capture overlay basename (e.g. ext3capture:basename=SIZE where size is 12G or 12M). - In the case of a Data Provider, the overlays will be the files that the provider uses. - Each overlay specification should be on a separate line. - You can add comments, indicated with hash symbol #. +

+ Each overlay or bindmount specification should be on a separate line. +

+ An overlay specification can be either: +

+

+

+ In the case of a Data Provider, the overlays will be the SquashFS files that the provider uses for its storage. The provider of course must be local to the current execution server. +

+ A bindmount specification is one of: +

+

+ You can add comments, indicated with a hash symbol #. For example, file:/a/b/atlas.squashfs # brain atlas

<% end %>