Add GIT_COMMIT, GIT_SHA_SHORT and GIT_BRANCH variables

Adds variables containing git commit, branch when the workflow resides
within a git repository folder. Variables are defined and available to
steps within their corresponding container.

cli/popper/config.py

@@ -21,10 +21,21 @@ def __init__(self, engine_name=None, resman_name=None, config_file=None,
         self.skip_pull = skip_pull
         self.skip_clone = skip_clone
 
-        if os.path.isdir(self.workspace_dir):
-            self.repo = scm.new_repo(self.workspace_dir)
+        # attempt to create a git.Repo. If folder doesn't exist, we get None
+        self.repo = scm.new_repo(self.workspace_dir)
+
+        if self.repo:
+            sha = self.repo.head.object.hexsha
+            self.git_commit = self.repo.git.rev_parse(sha)
+            self.git_sha_short = self.repo.git.rev_parse(sha, short=7)
+            if self.repo.head.is_detached:
+                self.git_branch = self.git_sha_short
+            else:
+                self.git_branch = self.repo.active_branch.name
         else:
-            self.repo = None
+            self.git_commit = 'na'
+            self.git_sha_short = 'na'
+            self.git_branch = 'na'
 
         wid = shake_256(self.workspace_dir.encode('utf-8')).hexdigest(4)
         self.wid = wid

cli/popper/runner.py

@@ -35,7 +35,8 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc, traceback):
         """calls __exit__ on all instantiated step runners"""
-        self._config.repo.close()
+        if self._config.repo:
+            self._config.repo.close()
         for _, r in WorkflowRunner.__runners.items():
             r.__exit__(exc_type, exc, traceback)
         WorkflowRunner.__runners = {}
@@ -211,17 +212,19 @@ def _step_runner(self, engine_name, step):
 class StepRunner(object):
     """Base class for step runners, assumed to be singletons."""
 
-    def __init__(self, config=PopperConfig()):
-        self._config = config
+    def __init__(self, config=None):
+        if not config:
+            self._config = PopperConfig()
+        else:
+            self._config = config
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc, traceback):
         pass
 
-    @staticmethod
-    def prepare_environment(step, env={}):
+    def _prepare_environment(self, step, env={}):
         """Prepare environment variables for a step, which includes those in
         the 'env' and 'secrets' attributes.
 
@@ -236,6 +239,14 @@ def prepare_environment(step, env={}):
         for s in step.get('secrets', []):
             step_env.update({s: os.environ[s]})
         step_env.update(env)
+
+        # define GIT_* variables
+        if self._config.repo:
+            step_env.update({
+                'GIT_COMMIT': self._config.git_commit,
+                'GIT_BRANCH': self._config.git_branch,
+                'GIT_SHA_SHORT': self._config.git_sha_short,
+            })
         return step_env
 
     def stop_running_tasks(self):

cli/popper/runner_host.py

@@ -34,7 +34,7 @@ def __exit__(self, exc_type, exc, traceback):
         pass
 
     def run(self, step):
-        step_env = StepRunner.prepare_environment(step, os.environ)
+        step_env = self._prepare_environment(step, os.environ)
 
         cmd = step.get('runs', [])
         if not cmd:
@@ -173,7 +173,7 @@ def _get_build_info(self, step):
             build = False
         elif './' in step['uses']:
             img = f'{pu.sanitized_name(step["name"], "step")}'
-            tag = f'{self._config.workspace_sha}'
+            tag = f'{self._config.git_sha_short}'
             build_ctx_path = os.path.join(self._config.workspace_dir,
                                           step['uses'])
         else:
@@ -225,7 +225,7 @@ def _get_container_kwargs(self, step, img, name):
                 '/var/run/docker.sock:/var/run/docker.sock'
             ],
             "working_dir": '/workspace',
-            "environment": StepRunner.prepare_environment(step),
+            "environment": self._prepare_environment(step),
             "entrypoint": step.get('runs', None),
             "detach": True
         }
@@ -418,7 +418,7 @@ def _create_container(self, step, cid):
                     pull_folder=self._singularity_cache)
 
     def _singularity_start(self, step, cid):
-        env = StepRunner.prepare_environment(step)
+        env = self._prepare_environment(step)
 
         # set the environment variables
         for k, v in env.items():

cli/popper/runner_slurm.py

@@ -192,13 +192,12 @@ def run(self, step):
         return ecode
 
     def _create_cmd(self, step, cid):
-        env = SlurmRunner.prepare_environment(step)
+        env = self._prepare_environment(step)
         for k, v in env.items():
             os.environ[k] = str(v)
 
         args = step.get('args', '')
         runs = step.get('runs', '')
-        ecode = None
 
         if runs:
             commands = runs

cli/popper/scm.py

@@ -16,7 +16,7 @@ def new_repo(gitrepo_dir=None):
     Returns:
         git.Repo: repo object or None if '.git/' not found in any parent folder
     """
-    if not gitrepo_dir:
+    if not gitrepo_dir or not os.path.isdir(gitrepo_dir):
         return None
     try:
         return git.Repo(gitrepo_dir, search_parent_directories=True)

cli/test/test_cmd_scaffold.py

@@ -39,8 +39,7 @@ def test_scaffold(self):
                         'needs': ['1']}})
 
         with self.assertLogs('popper') as test_logger:
-
-            result = runner.invoke(cmd_run.cli, ['-f', file_loc])
+            result = runner.invoke(cmd_run.cli, ['-f', file_loc, '-w', wf_dir])
             self.assertEqual(result.exit_code, 0)
             self.assertTrue(len(test_logger.output))
             self.assertTrue(

cli/test/test_config.py

@@ -1,11 +1,12 @@
-import unittest
 import os
 
 from popper.config import PopperConfig
 from popper.cli import log
 
+from .test_common import PopperTest
 
-class TestPopperConfig(unittest.TestCase):
+
+class TestPopperConfig(PopperTest):
     default_args = {
         'skip_clone': False,
         'engine_name': 'docker',
@@ -50,6 +51,20 @@ def test_config_non_defaults(self):
         self.assertEqual(expected,
                          TestPopperConfig.extract_dict(expected, actual))
 
+    def test_config_without_git_repo(self):
+        conf = PopperConfig(workspace_dir='/tmp/foo')
+        self.assertEqual('na', conf.git_commit)
+        self.assertEqual('na', conf.git_branch)
+        self.assertEqual('na', conf.git_sha_short)
+
+    def test_config_with_git_repo(self):
+        r = self.mk_repo()
+        conf = PopperConfig(workspace_dir=r.working_dir)
+        sha = r.head.object.hexsha
+        self.assertEqual(r.git.rev_parse(sha), conf.git_commit)
+        self.assertEqual(r.git.rev_parse(sha, short=7), conf.git_sha_short)
+        self.assertEqual(r.active_branch.name, conf.git_branch)
+
     def test_config_from_file(self):
         config = {
             'engine': {'options': {'privileged': True}},

cli/test/test_runner.py

@@ -3,11 +3,13 @@
 import shutil
 
 from unittest.mock import patch
+
+from popper.cli import log
 from popper.config import PopperConfig
 from popper.parser import YMLWorkflow
 from popper.runner import WorkflowRunner, StepRunner
 
-from popper.cli import log
+from .test_common import PopperTest
 
 
 class TestWorkflowRunner(unittest.TestCase):
@@ -85,35 +87,54 @@ def test_clone_repos(self):
 
     def test_steprunner_factory(self):
         with WorkflowRunner(PopperConfig()) as r:
-            self.assertEqual(r._step_runner('host', None).__class__.__name__,
-                             'HostRunner')
-            self.assertEqual(r._step_runner('docker', None).__class__.__name__,
-                             'DockerRunner')
+            self.assertEqual('HostRunner',
+                             r._step_runner('host', None).__class__.__name__)
+            self.assertEqual('DockerRunner',
+                             r._step_runner('docker', None).__class__.__name__)
 
     def test_setup_base_cache(self):
         cache_dir = WorkflowRunner._setup_base_cache()
         try:
-            self.assertEqual(cache_dir, os.environ['XDG_CACHE_HOME'])
+            self.assertEqual(os.environ['XDG_CACHE_HOME'], cache_dir)
         except KeyError:
             self.assertEqual(
-                cache_dir,
                 os.path.join(
                     os.environ['HOME'],
-                    '.cache/popper'))
+                    '.cache/popper'),
+                cache_dir)
 
         os.environ['POPPER_CACHE_DIR'] = '/tmp/popper'
         cache_dir = WorkflowRunner._setup_base_cache()
-        self.assertEqual(cache_dir, '/tmp/popper')
+        self.assertEqual('/tmp/popper', cache_dir)
         os.environ.pop('POPPER_CACHE_DIR')
 
 
-class TestStepRunner(unittest.TestCase):
+class TestStepRunner(PopperTest):
     def setUp(self):
         log.setLevel('CRITICAL')
 
-    def test_prepare_environment(self):
-        step = {'name': 'a', 'env': {'FOO': 'BAR'}, 'secrets': ['A']}
-        os.environ['A'] = 'BC'
-        env = StepRunner.prepare_environment(step, {'another': 'b'})
-        self.assertDictEqual(env, {'FOO': 'BAR', 'A': 'BC', 'another': 'b'})
-        os.environ.pop('A')
+    def test_prepare_environment_without_git(self):
+        with StepRunner(PopperConfig(workspace_dir='/tmp/foo')) as r:
+            step = {'name': 'a', 'env': {'FOO': 'BAR'}, 'secrets': ['A']}
+            os.environ['A'] = 'BC'
+            env = r._prepare_environment(step, {'other': 'b'})
+            self.assertDictEqual({'FOO': 'BAR', 'A': 'BC', 'other': 'b'}, env)
+            os.environ.pop('A')
+
+    def test_prepare_environment_with_git(self):
+        repo = self.mk_repo()
+        conf = PopperConfig(workspace_dir=repo.working_dir)
+        with StepRunner(conf) as r:
+            step = {'name': 'a', 'env': {'FOO': 'BAR'}, 'secrets': ['A']}
+            os.environ['A'] = 'BC'
+            env = r._prepare_environment(step, {'other': 'b'})
+            expected = {
+                'FOO': 'BAR',
+                'A': 'BC',
+                'other': 'b',
+                'GIT_COMMIT': conf.git_commit,
+                'GIT_BRANCH': conf.git_branch,
+                'GIT_SHA_SHORT': conf.git_sha_short,
+            }
+            self.assertDictEqual(expected, env)
+            os.environ.pop('A')

cli/test/test_runner_host.py

@@ -103,7 +103,8 @@ class TestHostDockerRunner(PopperTest):
     def setUp(self):
         log.setLevel('CRITICAL')
 
-    @unittest.skipIf(os.environ['ENGINE'] != 'docker', 'ENGINE != docker')
+    @unittest.skipIf(os.environ.get('ENGINE', 'docker') != 'docker',
+                     'ENGINE != docker')
     def test_create_container(self):
         config = PopperConfig()
         step = {
@@ -117,7 +118,8 @@ def test_create_container(self):
             self.assertEqual(c.status, 'created')
             c.remove()
 
-    @unittest.skipIf(os.environ['ENGINE'] != 'docker', 'ENGINE != docker')
+    @unittest.skipIf(os.environ.get('ENGINE', 'docker') != 'docker',
+                     'ENGINE != docker')
     def test_stop_running_tasks(self):
         with DockerRunner() as dr:
             dclient = docker.from_env()
@@ -132,7 +134,8 @@ def test_stop_running_tasks(self):
             self.assertEqual(c2.status, 'created')
             dclient.close()
 
-    @unittest.skipIf(os.environ['ENGINE'] != 'docker', 'ENGINE != docker')
+    @unittest.skipIf(os.environ.get('ENGINE', 'docker') != 'docker',
+                     'ENGINE != docker')
     def test_get_container_kwargs(self):
         step = {
             'uses': 'popperized/bin/sh@master',
@@ -181,7 +184,8 @@ def test_get_container_kwargs(self):
                 'hostname': 'popper.local',
                 'domainname': 'www.example.org'})
 
-    @unittest.skipIf(os.environ['ENGINE'] != 'docker', 'ENGINE != docker')
+    @unittest.skipIf(os.environ.get('ENGINE', 'docker') != 'docker',
+                     'ENGINE != docker')
     def test_get_build_info(self):
         step = {
             'uses': 'popperized/bin/sh@master',
@@ -212,7 +216,8 @@ def test_get_build_info(self):
             self.assertEqual(tag, '3.9')
             self.assertEqual(build_sources, None)
 
-    @unittest.skipIf(os.environ['ENGINE'] != 'docker', 'ENGINE != docker')
+    @unittest.skipIf(os.environ.get('ENGINE', 'docker') != 'docker',
+                     'ENGINE != docker')
     def test_docker_basic_run(self):
 
         repo = self.mk_repo()
@@ -260,7 +265,7 @@ def setUp(self):
         log.setLevel('CRITICAL')
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_get_recipe_file(self):
         repo = self.mk_repo()
@@ -299,7 +304,7 @@ def test_get_recipe_file(self):
                           build_ctx_path, 'sample.sif')
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_create_container(self):
         config = PopperConfig()
@@ -343,7 +348,7 @@ def test_create_container(self):
             os.remove(os.path.join(sr._singularity_cache, cid_two))
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_setup_singularity_cache(self):
         config = PopperConfig()
@@ -355,7 +360,7 @@ def test_setup_singularity_cache(self):
                 sr._singularity_cache)
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_get_container_options(self):
         config_dict = {
@@ -387,7 +392,7 @@ def test_get_container_options(self):
                 '--ipc'])
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_get_build_info(self):
         step = {
@@ -418,7 +423,7 @@ def test_get_build_info(self):
             self.assertEqual(build_sources, None)
 
     @unittest.skipIf(
-        os.environ['ENGINE'] != 'singularity',
+        os.environ.get('ENGINE', 'docker') != 'singularity',
         'ENGINE != singularity')
     def test_singularity_start(self):
         repo = self.mk_repo()