adding support for spack monitor with containerize

Signed-off-by: Vanessa Sochat <sochat1@llnl.gov>

lib/spack/docs/monitoring.rst

@@ -92,3 +92,139 @@ added to your monitoring database. This setting will not be applied to the
 first request to check if the server is running, but to subsequent requests.
 If you don't have a monitor server running and you want to build, simply
 don't provide the ``--monitor`` flag!
+
+
+----------------------------
+Monitoring with Containerize
+----------------------------
+
+The same argument group is available to add to a containerize command. 
+
+^^^^^^
+Docker
+^^^^^^
+
+To add monitoring to a Docker container recipe generation using the defaults,
+and assuming a monitor server running on localhost, you would
+start with a spack.yaml in your present working directory:
+
+.. code-block:: yaml
+
+   spack:
+     specs:
+       - samtools
+
+And then do:
+
+.. code-block:: console
+
+    # preview first
+    spack containerize --monitor
+    
+    # and then write to a Dockerfile
+    spack containerize --monitor > Dockerfile
+    
+    
+The install command will be edited to include commands for enabling monitoring.
+However, getting secrets into the container for your monitor server is something
+that should be done carefully. Specifically you should:
+
+ - Never try to define secrets as ENV, ARG, or using ``--build-arg``
+ - Do not try to get the secret into the container via a "temporary" file that you remove (it in fact will still exist in a layer)
+
+Instead, it's recommended to use buildkit `as explained here <https://pythonspeed.com/articles/docker-build-secrets/>`_.
+You'll need to again export environment variables for your spack monitor server:
+
+.. code-block:: console
+
+    $ export SPACKMON_TOKEN=50445263afd8f67e59bd79bff597836ee6c05438
+    $ export SPACKMON_USER=spacky
+
+And then use buildkit along with your build and identifying the name of the secret:
+
+.. code-block:: console
+
+    $ DOCKER_BUILDKIT=1 docker build --secret id=st,env=SPACKMON_TOKEN --secret id=su,env=SPACKMON_USER -t spack/container . 
+    
+The secrets are expected to come from your environment, and then will be temporarily mounted and available
+at ``/run/secrets/<name>``. If you forget to supply them (and authentication is required) the build
+will fail. If you need to build on your host (and interact with a spack monitor at localhost) you'll
+need to tell Docker to use the host network:
+
+.. code-block:: console
+
+    $ DOCKER_BUILDKIT=1 docker build --network="host" --secret id=st,env=SPACKMON_TOKEN --secret id=su,env=SPACKMON_USER -t spack/container . 
+    
+
+^^^^^^^^^^^
+Singularity
+^^^^^^^^^^^
+
+To add monitoring to a Singularity container build, the spack.yaml needs to
+be modified slightly to specify wanting a different format:
+
+
+.. code-block:: yaml
+
+   spack:
+     specs:
+       - samtools
+     container:
+       format: singularity
+       
+       
+Again, generate the recipe:
+
+
+.. code-block:: console
+
+    # preview first
+    $ spack containerize --monitor
+    
+    # then write to a Singularity recipe
+    $ spack containerize --monitor > Singularity
+
+
+Singularity doesn't have a direct way to define secrets at build time, so we have
+to do a bit of a manual command to add a file, source secrets in it, and remove it.
+Since Singularity doesn't have layers like Docker, deleting a file will truly
+remove it from the container and history. So let's say we have this file,
+``secrets.sh``:
+
+.. code-block:: console
+
+    # secrets.sh
+    export SPACKMON_USER=spack
+    export SPACKMON_TOKEN=50445263afd8f67e59bd79bff597836ee6c05438
+
+
+We would then generate the Singularity recipe, and add a files section,
+a source of that file at the start of ``%post``, and **importantly**
+a removal of the final at the end of that same section.
+
+.. code-block:: console
+
+    Bootstrap: docker
+    From: spack/ubuntu-bionic:latest
+    Stage: build
+
+    %files
+      secrets.sh /opt/secrets.sh 
+
+    %post
+      . /opt/secrets.sh
+
+      # spack install commands are here
+      ...
+      
+      # Don't forget to remove here!
+      rm /opt/secrets.sh
+
+
+You can then build the container as your normally would.
+
+.. code-block::
+
+    $ sudo singularity build container.sif Singularity 
+
+

lib/spack/spack/cmd/containerize.py

@@ -5,13 +5,18 @@
 import os
 import os.path
 import spack.container
+import spack.monitor
 
 description = ("creates recipes to build images for different"
                " container runtimes")
 section = "container"
 level = "long"
 
 
+def setup_parser(subparser):
+    monitor_group = spack.monitor.get_monitor_group(subparser)  # noqa
+
+
 def containerize(parser, args):
     config_dir = args.env_dir or os.getcwd()
     config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml'))
@@ -21,5 +26,12 @@ def containerize(parser, args):
 
     config = spack.container.validate(config_file)
 
+    # If we have a monitor request, add monitor metadata to config
+    if args.use_monitor:
+        config['spack']['monitor'] = {"disable_auth": args.monitor_disable_auth,
+                                      "host": args.monitor_host,
+                                      "keep_going": args.monitor_keep_going,
+                                      "prefix": args.monitor_prefix,
+                                      "tags": getattr(args, "monitor_tags", None)}
     recipe = spack.container.recipe(config)
     print(recipe)

lib/spack/spack/cmd/install.py

@@ -345,6 +345,10 @@ def get_tests(specs):
                 reporter.filename = default_log_file(specs[0])
             reporter.specs = specs
 
+            # Tell the monitor about the specs
+            if args.use_monitor and specs:
+                monitor.new_configuration(specs)
+
             tty.msg("Installing environment {0}".format(env.name))
             with reporter('build'):
                 env.install_all(args, **kwargs)

lib/spack/spack/container/writers/__init__.py

@@ -110,13 +110,36 @@ def paths(self):
             view='/opt/view'
         )
 
+    @tengine.context_property
+    def monitor(self):
+        """Enable using spack monitor during build."""
+        Monitor = collections.namedtuple('Monitor', [
+            'enabled', 'host', 'disable_auth', 'prefix', 'keep_going', 'tags'
+        ])
+        monitor = self.config.get("monitor")
+
+        # If we don't have a monitor group, cut out early.
+        if not monitor:
+            return Monitor(False, None, None, None, None)
+
+        return Monitor(
+            enabled=True,
+            host=monitor.get('host'),
+            prefix=monitor.get('prefix'),
+            disable_auth=monitor.get("disable_auth"),
+            keep_going=monitor.get("keep_going"),
+            tags=monitor.get('tags')
+        )
+
     @tengine.context_property
     def manifest(self):
         """The spack.yaml file that should be used in the image"""
         import jsonschema
         # Copy in the part of spack.yaml prescribed in the configuration file
         manifest = copy.deepcopy(self.config)
         manifest.pop('container')
+        if "monitor" in manifest:
+            manifest.pop("monitor")
 
         # Ensure that a few paths are where they need to be
         manifest.setdefault('config', syaml.syaml_dict())

lib/spack/spack/monitor.py

@@ -244,6 +244,12 @@ def issue_request(self, request, retry=True):
             elif hasattr(e, 'code'):
                 msg = e.code
 
+            # If we can parse the message, try it
+            try:
+                msg += "\n%s" % e.read().decode("utf8", 'ignore')
+            except Exception:
+                pass
+
             if self.allow_fail:
                 tty.warning("Request to %s was not successful, but continuing." % e.url)
                 return

share/spack/spack-completion.bash

@@ -694,7 +694,7 @@ _spack_config_revert() {
 }
 
 _spack_containerize() {
-    SPACK_COMPREPLY="-h --help"
+    SPACK_COMPREPLY="-h --help --monitor --monitor-no-auth --monitor-keep-going --monitor-host --monitor-prefix"
 }
 
 _spack_create() {

share/spack/templates/container/Dockerfile

@@ -14,7 +14,7 @@ RUN mkdir {{ paths.environment }} \
 {{ manifest }} > {{ paths.environment }}/spack.yaml
 
 # Install the software, remove unnecessary deps
-RUN cd {{ paths.environment }} && spack env activate . && spack install --fail-fast && spack gc -y
+RUN {% if monitor.enabled %}--mount=type=secret,id=su --mount=type=secret,id=st{% endif %} cd {{ paths.environment }} && spack env activate . {% if not monitor.disable_auth %}&& export SPACKMON_USER=$(cat /run/secrets/su) && export SPACKMON_TOKEN=$(cat /run/secrets/st) {% endif %}&& spack install {% if monitor.enabled %}--monitor {% if monitor.prefix %}--monitor-prefix {{ monitor.prefix }} {% endif %}{% if monitor.tags %}--monitor-tags {{ monitor.tags }} {% endif %}{% if monitor.keep_going %}--monitor-keep-going {% endif %}{% if monitor.host %}--monitor-host {{ monitor.host }} {% endif %}{% if monitor.disable_auth %}--monitor-disable-auth {% endif %}{% endif %}--fail-fast && spack gc -y
 {% if strip %}
 
 # Strip all the binaries

share/spack/templates/container/singularity.def

@@ -21,7 +21,7 @@ EOF
   # Install all the required software
   . /opt/spack/share/spack/setup-env.sh
   spack env activate .
-  spack install --fail-fast
+  spack install {% if monitor.enabled %}--monitor {% if monitor.prefix %}--monitor-prefix {{ monitor.prefix }} {% endif %}{% if monitor.tags %}--monitor-tags {{ monitor.tags }} {% endif %}{% if monitor.keep_going %}--monitor-keep-going {% endif %}{% if monitor.host %}--monitor-host {{ monitor.host }} {% endif %}{% if monitor.disable_auth %}--monitor-disable-auth {% endif %}{% endif %}--fail-fast
   spack gc -y
   spack env deactivate
   spack env activate --sh -d . >> {{ paths.environment }}/environment_modifications.sh