Bindmount

BrainPortal/app/controllers/sessions_controller.rb

@@ -42,7 +42,7 @@ def new #:nodoc:
     # HEAD requests render nothing
     req_method = request.method.to_s.upcase
     if req_method == 'HEAD'
-      render :plain => "", :status => :ok
+      head :ok
       return
     end
 

BrainPortal/app/models/cluster_task.rb

@@ -2373,7 +2373,7 @@ def singularity_commands(command_script)
     # must be on a device different from the one for the work directory.
     capture_basenames = ext3capture_basenames.map { |basename,_| basename }
 
-    # (4) More -B (bind mounts) for all the relevant local data providers.
+    # (4a) More -B (bind mounts) for all the relevant local data providers.
     # This will be a string "-B path1 -B path2 -B path3" etc.
     # In the case of read-only input files, ro option is added
     esc_local_dp_mountpoints = local_dp_storage_paths.inject("") do |sing_opts,path|
@@ -2382,6 +2382,12 @@ def singularity_commands(command_script)
       sing_opts
     end
 
+    # (4b) Add tool config bindmounts, specified in 'overlay'
+    esc_local_bindmounts = bindmount_paths.inject("") do |sing_opts,(from_path,cont_path)|
+      sing_opts += " -B #{from_path.bash_escape}:#{cont_path.bash_escape}"
+      sing_opts
+    end
+
     # (5) Overlays defined in the ToolConfig
     # Some of them might be patterns (e.g. /a/b/data*.squashfs) that need to
     # be resolved locally.
@@ -2497,7 +2503,8 @@ def singularity_commands(command_script)
 #   a) at its original cluster full path
 #   b) at /DP_Cache (used only when shortening workdir)
 # 3) we mount the root of the gridshare area (for all tasks)
-# 4) we mount each (if any) of the root directories for local data providers
+# 4a) we mount each (if any) of the root directories for local data providers
+# 4b) we mount each (if any) of the bindmounts configured in the ToolConfig
 # 5) we mount (if any) other fixed file system overlays
 # 6) we mount (if any) capture ext3 filesystems
 # 7) with -H we set the task's work directory as the singularity $HOME directory
@@ -2508,6 +2515,7 @@ def singularity_commands(command_script)
     -B #{cache_dir.bash_escape}:/DP_Cache       \\
     -B #{gridshare_dir.bash_escape}             \\
     #{esc_local_dp_mountpoints}                 \\
+    #{esc_local_bindmounts}                     \\
     #{overlay_mounts}                           \\
     -B #{task_workdir.bash_escape}:#{effect_workdir.bash_escape} \\
     #{esc_capture_mounts}                       \\
@@ -2566,6 +2574,12 @@ def ext3capture_basenames
     self.tool_config.ext3capture_basenames
   end
 
+  # Just invokes the same method on the task's ToolConfig.
+  # Returns an array of pairs, e.g. [ [ src, dest ], [ src, dest ] ]
+  def bindmount_paths
+    self.tool_config.bindmount_paths
+  end
+
   # This method creates an empty +filename+ with +size+ bytes
   # (where size is specified like what the unix 'truncate' command accepts)
   # and then formats it with a ext3 filesystem. If the filename already exists,

BrainPortal/app/models/tool_config.rb

@@ -384,8 +384,10 @@ def cbrain_task_class
   #         dp:1234
   #      # CBRAIN db registered file
   #         userfile:1234
-  #      # A ext3 capture filesystem, will NOT be returned here as an overlay
+  #      # A ext3 capture filesystem, will NOT be returned here as an overlay (see method ext3capture_basenames() instead)
   #         ext3capture:basename=12G
+  #      # A bind mount, will NOT be returned here as an overlay (see method bindmount_paths() instead)
+  #        bindmount:/local/basename:/container/basename
   def singularity_overlays_full_paths
     specs = parsed_overlay_specs
     specs.map do |knd, id_or_name|
@@ -410,14 +412,16 @@ def singularity_overlays_full_paths
         { userfile.cache_full_path() =>  "registered userfile" }
       when 'ext3capture'
         []  # handled separately
+      when 'bindmount'
+        []  # handled separately
       else
         cb_error "Invalid '#{knd}:#{id_or_name}' overlay."
       end
     end.flatten.reduce(&:merge) || []
   end
 
   # Returns an array of the data providers that are
-  # specified in the attribute singularity_overlays_specs,
+  # specified in the attribute +singularity_overlays_specs+,
   # ignoring all other overlay specs for normal files.
   def data_providers_with_overlays
     return @_data_providers_with_overlays_ if @_data_providers_with_overlays_
@@ -428,6 +432,39 @@ def data_providers_with_overlays
     end.compact
   end
 
+  # Returns an array of pairs extracted from the attribute
+  # +singularity_overlays_specs+ , ignoring all other overlay
+  # specs for normal files.
+  def bindmount_paths
+    specs = parsed_overlay_specs.presence || []
+
+    # Pairs of paths obtained from the ToolConfig's "overlay" configuration.
+    tc_paths = specs
+      .map { |pair| pair[1] if pair[0] == 'bindmount' }
+      .compact
+      .map { |frompath_contpath| frompath_contpath.split(":",2) }
+
+    # One additional bindmount path from the plugins directory where the tool
+    # comes from. Available ONLY if tool is configured with
+    # a boutiques descriptor. The following lines of code make
+    # a bunch of checks and as soon as a check fails, we just
+    # return with the array tc_paths computed above.
+    descriptor = self.boutiques_descriptor rescue nil
+    return tc_paths if descriptor.blank?
+    container_mountpoint = descriptor.custom["cbrain:plugins-container-bindmount"]
+    return tc_paths if container_mountpoint.blank?
+    desc_file = descriptor.from_file         # "/path/to/RailsApp/cbrain-plugins/installed_plugins/boutiques_descriptors/toolname.json"
+    return tc_paths if desc_file.blank?
+    real_path = File.realpath(desc_file)     # "/path/to/RailsApp/cbrain-plugins/plugin-name/boutiques_descriptors/toolname.json"
+    parent1 = Pathname.new(real_path).parent # "/path/to/RailsApp/cbrain-plugins/plugin-name/boutiques_descriptors"
+    return tc_paths if parent1.basename.to_s != "boutiques_descriptors" # check plugins convention
+    plugin_path = parent1.parent             # "/path/to/RailsApp/cbrain-plugins/plugin-name"
+    plugin_containerized_dir = plugin_path + "container_mnt" # special plugins folder to mount
+    return tc_paths if ! File.directory?(plugin_containerized_dir.to_s)
+    tc_paths << [ plugin_containerized_dir.to_s, container_mountpoint + ":ro" ]
+    return tc_paths
+  end
+
   # Returns pairs [ [ basename, size], ...] as in [ [ 'work', '28g' ]
   def ext3capture_basenames
     specs = parsed_overlay_specs
@@ -475,7 +512,8 @@ def validate_container_rules #:nodoc:
     return errors.empty?
   end
 
-  # breaks down overlay spec onto a list of overlays
+  # Breaks down the singularity_overlays_specs attribute, and returns a list of pairs [ type, value ] e.g.
+  #   [ [ 'userfile', '1234' ], [ 'file', '/hello/bye/data.sqs' ], [ 'bindmount', '/some/data/dir:/mount/this/here' ] ]
   def parsed_overlay_specs
     specs = self.singularity_overlays_specs
     return [] if specs.blank?
@@ -486,8 +524,9 @@ def parsed_overlay_specs
   end
 
   # Verifies that the admin has entered a set of
-  # overlay specifications properly. One or several of:
+  # overlay and bindmount specifications properly. One or several of:
   #
+  #    #### the overlay rules
   #    file:/full/path/to/something.squashfs
   #    file:/full/path/to/pattern*/data?.squashfs
   #    userfile:333
@@ -497,6 +536,8 @@ def parsed_overlay_specs
   #    ext3capture:work=30G
   #    ext3capture:tool_1.1.2=15M
   #
+  #    bindmount:/bourreau/path/to:/container/path/to     # bindmount rules ( additionally introduced )
+  #
   def validate_overlays_specs #:nodoc:
     specs = parsed_overlay_specs
 
@@ -542,6 +583,10 @@ def validate_overlays_specs #:nodoc:
           self.errors.add(:singularity_overlays_specs, "contains invalid ext3capture specification (must be like ext3capture:basename=1g or 2m etc)")
         end
 
+      when 'bindmount' # this is for binding to container rather than overlays
+        if id_or_name !~ /\A\/\w[\w\.\-\/]+:\/\w[\w\.\-\/]+(:ro)?\z/
+          self.errors.add(:singularity_overlays_specs, "contains invalid bindmount specification (must be like 'bindmount:/path/in/bourreau:/path/in/container' with or without a final ':ro') and free from special characters")
+        end
       else
         # Other errors
         self.errors.add(:singularity_overlays_specs, "contains invalid specification '#{kind}:#{id_or_name}'")

BrainPortal/app/views/bourreaux/_bourreaux_display.html.erb

@@ -407,7 +407,7 @@
             Stopping an Execution Server will also stop the Task Workers <strong>and</strong> Activity Workers running on it.
             Make sure they are not actively <%= html_colorize("processing","orange") %> something.
           </p>
-          <%= external_submit_button "2. Stop Bourreaux",     "bourreau_form",
+          <%= external_submit_button "2. Stop Execution Server", "bourreau_form",
               :class => 'button',
               :name  => 'operation',
               :value => 'stop_bourreaux',
@@ -421,7 +421,7 @@
               :class => 'button',
               :name  => 'operation',
               :value => 'stop_tunnels',
-              :data  => { :confirm => "Make sure Bourreaux, Task Workers and Activity workers are all stopped!" }
+              :data  => { :confirm => "Make sure Execution Servers, Task Workers and Activity workers are all stopped!" }
           %>
         </div>
       <% end %>

BrainPortal/app/views/tool_configs/_form_fields.html.erb

@@ -222,17 +222,30 @@
     <% t.edit_cell(:singularity_overlays_specs, :header => "Singularity Overlays", :show_width => 2, :content => full_description(@tool_config.singularity_overlays_specs)) do |f| %>
       <%= f.text_area :singularity_overlays_specs, :rows => 6, :cols => 120 %>
         <div class="wide_field_explanation">
-        This field can contain one or several specifications for data overlays
+        This field can contain one or several specifications for data overlays and bindmounts
         to be included when the task is started with Singularity.
-        A specification can be either
-        a full path (e.g. <em>file:/a/b/data.squashfs</em>),
-        a path with a pattern (e.g. <em>file:/a/b/data*.squashfs</em>),
-        a registered file identified by ID (e.g. <em>userfile:123</em>),
-        a SquashFS Data Provider identified by its ID or name (e.g. <em>dp:123</em>, <em>dp:DpNameHere</em>)
-        or an ext3 capture overlay basename (e.g. <em>ext3capture:basename=SIZE</em> where size is <em>12G</em> or <em>12M</em>).
-        In the case of a Data Provider, the overlays will be the files that the provider uses.
-        Each overlay specification should be on a separate line.
-        You can add comments, indicated with hash symbol <em>#</em>.
+        <p>
+        Each overlay or bindmount specification should be on a separate line.
+        <p>
+        An overlay specification can be either:
+        <p>
+        <ul>
+        <li>a full path (e.g. <em>file:/a/b/data.squashfs</em>),</li>
+        <li>a path with a pattern (e.g. <em>file:/a/b/data*.squashfs</em>),</li>
+        <li>a registered file identified by ID (e.g. <em>userfile:123</em>),</li>
+        <li>a SquashFS Data Provider identified by its ID or name (e.g. <em>dp:123</em>, <em>dp:DpNameHere</em>)</li>
+        <li>or an ext3 capture overlay basename (e.g. <em>ext3capture:basename=SIZE</em> where size is <em>12G</em> or <em>12M</em>).</li>
+        </ul>
+        <p>
+        In the case of a Data Provider, the overlays will be the SquashFS files that the provider uses for its storage. The provider of course must be local to the current execution server.
+        <p>
+        A bindmount specification is one of:
+        <ul>
+          <li><em>bindmount:/bourreau/path/to/data:/containerized/path/to/data</em> or</li>
+          <li><em>bindmount:/bourreau/path/to/data:/containerized/path/to/data:ro</em></li>
+        </ul>
+        <p>
+        You can add comments, indicated with a hash symbol <em>#</em>.
         For example, <em>file:/a/b/atlas.squashfs # brain atlas</em>
         </div>
     <% end %>