Add utility to manipulate files en masse (#1166)

This PR adds the ability to pass in a configuration and manipulate files in bulk. Currently it just works for `copy` and `mkdir`.

A sample YAML file:
```
mkdir:
- ${CRMTEST}/output
copy:
- - ${CRMTEST}/fakefile1
  - ${CRMTEST}/output/fakefile1
- - ${CRMTEST}/fakefile2
  - ${CRMTEST}/output/fakefile2
```

To use it in a python script:
```
from pygw.utils import FileHandler
FileHandler(path='/export/emc-lw-cmartin/cmartin/work/tmp/test.yaml').sync()
```

ush/python/pygw/src/pygw/file_utils.py

@@ -0,0 +1,73 @@
+from .fsutils import cp, mkdir
+
+__all__ = ['FileHandler']
+
+
+class FileHandler:
+    """Class to manipulate files in bulk for a given configuration
+
+    Parameters
+    ----------
+    config : dict
+          A dictionary containing the "action" and the "act" in the form of a list
+
+    NOTE
+    ----
+    "action" can be one of mkdir", "copy", etc.
+    Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']]
+
+    Attributes
+    ----------
+    config : dict
+            Dictionary of files to manipulate
+    """
+
+    def __init__(self, config):
+
+        self.config = config
+
+    def sync(self):
+        """
+        Method to execute bulk actions on files described in the configuration
+        """
+        sync_factory = {
+            'copy': self._copy_files,
+            'mkdir': self._make_dirs,
+        }
+        # loop through the configuration keys
+        for action, files in self.config.items():
+            sync_factory[action](files)
+
+    @staticmethod
+    def _copy_files(filelist):
+        """Function to copy all files specified in the list
+
+        `filelist` should be in the form:
+        - [src, dest]
+
+        Parameters
+        ----------
+        filelist : list
+                List of lists of [src, dest]
+        """
+        for sublist in filelist:
+            if len(sublist) != 2:
+                raise Exception(
+                    f"List must be of the form ['src', 'dest'], not {sublist}")
+            src = sublist[0]
+            dest = sublist[1]
+            cp(src, dest)
+            print(f'Copied {src} to {dest}')  # TODO use logger
+
+    @staticmethod
+    def _make_dirs(dirlist):
+        """Function to make all directories specified in the list
+
+        Parameters
+        ----------
+        dirlist : list
+                List of directories to create
+        """
+        for dd in dirlist:
+            mkdir(dd)
+            print(f'Created {dd}')  # TODO use logger

ush/python/pygw/src/pygw/fsutils.py

@@ -3,7 +3,7 @@
 import shutil
 import contextlib
 
-__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p']
+__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p', 'cp']
 
 
 def mkdir_p(path):
@@ -45,3 +45,12 @@ def rm_p(path):
             pass
         else:
             raise OSError(f"unable to remove {path}")
+
+
+def cp(src, dest):
+    try:
+        shutil.copyfile(src, dest)
+    except OSError as exc:
+        raise OSError(f"unable to copy {src} to {dest}")
+    except FileNotFoundError as exc:
+        raise FileNotFoundError(exc)

ush/python/pygw/src/tests/test_file_utils.py

@@ -0,0 +1,66 @@
+import os
+from pygw.file_utils import FileHandler
+
+
+def test_mkdir(tmp_path):
+    """
+    Test for creating directories:
+    Parameters
+    ----------
+    tmp_path - pytest fixture
+    """
+
+    dir_path = tmp_path / 'my_test_dir'
+    d1 = f'{dir_path}1'
+    d2 = f'{dir_path}2'
+    d3 = f'{dir_path}3'
+
+    # Create config object for FileHandler
+    config = {'mkdir': [d1, d2, d3]}
+
+    # Create d1, d2, d3
+    FileHandler(config).sync()
+
+    # Check if d1, d2, d3 were indeed created
+    for dd in config['mkdir']:
+        assert os.path.exists(dd)
+
+
+def test_copy(tmp_path):
+    """
+    Test for copying files:
+    Parameters
+    ----------
+    tmp_path - pytest fixture
+    """
+
+    input_dir_path = tmp_path / 'my_input_dir'
+
+    # Create the input directory
+    config = {'mkdir': [input_dir_path]}
+    FileHandler(config).sync()
+
+    # Put empty files in input_dir_path
+    src_files = [input_dir_path / 'a.txt', input_dir_path / 'b.txt']
+    for ff in src_files:
+        ff.touch()
+
+    # Create output_dir_path and expected file names
+    output_dir_path = tmp_path / 'my_output_dir'
+    config = {'mkdir': [output_dir_path]}
+    FileHandler(config).sync()
+    dest_files = [output_dir_path / 'a.txt', output_dir_path / 'bb.txt']
+
+    copy_list = []
+    for src, dest in zip(src_files, dest_files):
+        copy_list.append([src, dest])
+
+    # Create config object for FileHandler
+    config = {'copy': copy_list}
+
+    # Copy input files to output files
+    FileHandler(config).sync()
+
+    # Check if files were indeed copied
+    for ff in dest_files:
+        assert os.path.isfile(ff)

ush/python/pygw/src/tests/test_template.py

@@ -114,9 +114,9 @@ def test_substitute_with_dependencies():
             'a': 1,
             'b': 2
         },
-        'dd': { '2': 'a', '1': 'b' },
-        'ee': { '3': 'a', '1': 'b' },
-        'ff': { '4': 'a', '1': 'b $(greeting)' },
+        'dd': {'2': 'a', '1': 'b'},
+        'ee': {'3': 'a', '1': 'b'},
+        'ff': {'4': 'a', '1': 'b $(greeting)'},
         'host': {
             'name': 'xenon',
             'config': '$(root)/hosts',
@@ -128,21 +128,20 @@ def test_substitute_with_dependencies():
         }
     }
     output = {'complex': {'a': 1, 'b': 2},
-                 'config': '/home/user/config/config.yaml',
-                 'config_file': 'config.yaml',
-                 'dd': {'1': 'b', '2': 'a'},
-                 'dictionary': {'a': 1, 'b': 2},
-                 'ee': {'1': 'b', '3': 'a'},
-                 'ff': {'1': 'b hello world', '4': 'a'},
-                 'greeting': 'hello world',
-                 'host': {'config': '/home/user/hosts',
-                          'config_file': '/home/user/config/config.yaml/xenon.config.yaml',
-                          'name': 'xenon',
-                          'proxy2': {'config': '/home/user/xenon.hello world.yaml',
-                                     'list': [['/home/user/xenon', 'toto.xenon.hello world'],
-                                              'config.yaml']}},
-                 'root': '/home/user',
-                 'world': 'world'}
-
+              'config': '/home/user/config/config.yaml',
+              'config_file': 'config.yaml',
+              'dd': {'1': 'b', '2': 'a'},
+              'dictionary': {'a': 1, 'b': 2},
+              'ee': {'1': 'b', '3': 'a'},
+              'ff': {'1': 'b hello world', '4': 'a'},
+              'greeting': 'hello world',
+              'host': {'config': '/home/user/hosts',
+                       'config_file': '/home/user/config/config.yaml/xenon.config.yaml',
+                       'name': 'xenon',
+                       'proxy2': {'config': '/home/user/xenon.hello world.yaml',
+                                  'list': [['/home/user/xenon', 'toto.xenon.hello world'],
+                                           'config.yaml']}},
+              'root': '/home/user',
+              'world': 'world'}
 
     assert Template.substitute_with_dependencies(input, input, TemplateConstants.DOLLAR_PARENTHESES) == output