Fix #75 add delete mechanisms (#76)

This adds two delete methods through the API and a newly created admin tool. Signed-off-by: David Brown <dmlb2000@gmail.com>
pacifica · Feb 19, 2019 · ac44ded · ac44ded
1 parent 2078225
commit ac44ded
Show file tree

Hide file tree

Showing 17 changed files with 268 additions and 115 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -8,7 +8,8 @@ stages:
 - pip install .
 - export ARCHIVEINTERFACE_CPCONFIG=$PWD/server.conf
 - cd tests
-- coverage run --include '*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pytest -v
+- coverage run --include '*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pacifica.archiveinterface --stop-after-a-moment
+- coverage run -a --include '*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pytest -xv
 - coverage report -m --fail-under 100
 jobs:
   include:

diff --git a/appveyor.yml b/appveyor.yml
@@ -24,5 +24,6 @@ test_script:
       pip install .;
       $env:ARCHIVEINTERFACE_CPCONFIG = "$PWD/server.conf";
       cd tests;
-      coverage run --include='*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pytest -v;
+      coverage run --include='*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pacifica.archiveinterface --stop-after-a-moment;
+      coverage run -a --include='*/site-packages/pacifica/archiveinterface/*' --omit '*/site-packages/pacifica/archiveinterface/backends/abstract/*' -m pytest -xv;
       coverage report -m --fail-under=100;
diff --git a/pacifica/archiveinterface/__main__.py b/pacifica/archiveinterface/__main__.py
@@ -7,7 +7,7 @@
 to support the new Backend Archie type
 
 """
-from os import path
+from os import path, environ
 from time import sleep
 from threading import Thread
 from argparse import ArgumentParser, SUPPRESS
@@ -76,3 +76,45 @@ def main():
         '/',
         args.cp_config
     )
+
+
+def cmd(argv=None):
+    """Command line admin tool for managing ingest."""
+    parser = ArgumentParser(description='Admin command line tool.')
+    parser.add_argument(
+        '-c', '--config', metavar='CONFIG', type=str, default=CONFIG_FILE,
+        dest='config', help='archiveinterface config file'
+    )
+    parser.add_argument(
+        '-t', '--type', dest='type', default='posix',
+        choices=['hpss', 'posix', 'hsmsideband'], help='use the typed backend'
+    )
+    parser.add_argument(
+        '--prefix', metavar='PREFIX', dest='prefix',
+        default='{}tmp'.format(path.sep), help='prefix to save data at'
+    )
+    subparsers = parser.add_subparsers(help='sub-command help')
+    delete_parser = subparsers.add_parser(
+        'delete', help='delete help', description='delete files')
+    delete_parser.add_argument(
+        'FILES', nargs='+',
+        help='delete the files'
+    )
+    delete_parser.set_defaults(func=delete_file)
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+def delete_file(args):
+    """Delete a file in the archive."""
+    environ['ARCHIVEINTERFACE_CONFIG'] = args.config
+    factory = ArchiveBackendFactory()
+    backend = factory.get_backend_archive(args.type, args.prefix)
+    for archfile in args.FILES:
+        backend.open(archfile, 'r')
+        backend.remove()
+    return 0
+
+
+if __name__ == '__main__':
+    main()
diff --git a/pacifica/archiveinterface/backends/abstract/archive.py b/pacifica/archiveinterface/backends/abstract/archive.py
@@ -100,3 +100,8 @@ def set_file_permissions(self):
     def patch(self, file_id, old_path):
         """Move a file."""
         pass
+
+    @abc.abstractmethod
+    def remove(self):
+        """Remove a file."""
+        pass
diff --git a/pacifica/archiveinterface/backends/hpss/archive.py b/pacifica/archiveinterface/backends/hpss/archive.py
@@ -254,3 +254,19 @@ def patch(self, file_id, old_path):
                 old_path, new_filepath, str(ex)
             )
             raise ArchiveInterfaceError(err_str)
+
+    def remove(self):
+        """Remove the file for an HPSS file."""
+        try:
+            if self._filepath:
+                buf_char_p = cast(self._filepath, c_char_p)
+                rcode = self._hpsslib.hpss_Chmod(buf_char_p, 0o644)
+                hpss_unlink = self._hpsslib.hpss_Unlink
+                hpss_unlink.restype = c_int
+                hpss_unlink.argtypes = [c_char_p]
+                rcode = hpss_unlink(buf_char_p)
+                self._check_rcode(rcode, 'Error removing hpss file: {}'.format(rcode))
+            self._filepath = None
+        except Exception as ex:
+            err_str = "Can't remove hpss file with error: " + str(ex)
+            raise ArchiveInterfaceError(err_str)
diff --git a/pacifica/archiveinterface/backends/oracle_hsm_sideband/archive.py b/pacifica/archiveinterface/backends/oracle_hsm_sideband/archive.py
@@ -6,6 +6,7 @@
 backend.
 """
 import os
+import stat
 import shutil
 from ...archive_utils import un_abs_path
 from ...config import get_config
@@ -153,3 +154,14 @@ def patch(self, file_id, old_path):
         except Exception as ex:
             err_str = "Can't move posix file with error: " + str(ex)
             raise ArchiveInterfaceError(err_str)
+
+    def remove(self):
+        """Remove the file for a posix file."""
+        try:
+            if self._filepath:
+                os.chmod(self._filepath, stat.S_IWRITE)
+                os.unlink(self._filepath)
+            self._filepath = None
+        except Exception as ex:
+            err_str = "Can't remove posix file with error: " + str(ex)
+            raise ArchiveInterfaceError(err_str)
diff --git a/pacifica/archiveinterface/backends/posix/archive.py b/pacifica/archiveinterface/backends/posix/archive.py
@@ -5,8 +5,8 @@
 Module that implements the abstract_backend_archive class for a posix
 backend.
 """
-
 import os
+import stat
 import shutil
 from ...archive_utils import un_abs_path, bytes_type
 from ...config import get_config
@@ -142,3 +142,14 @@ def patch(self, file_id, old_path):
         except Exception as ex:
             err_str = "Can't move posix file with error: " + str(ex)
             raise ArchiveInterfaceError(err_str)
+
+    def remove(self):
+        """Remove the file permissions for a posix file."""
+        try:
+            if self._filepath:
+                os.chmod(self._filepath, stat.S_IWRITE)
+                os.unlink(self._filepath)
+            self._filepath = None
+        except Exception as ex:
+            err_str = "Can't remove posix file with error: {}".format(ex)
+            raise ArchiveInterfaceError(err_str)
diff --git a/pacifica/archiveinterface/rest_generator.py b/pacifica/archiveinterface/rest_generator.py
@@ -126,3 +126,15 @@ def PATCH(self, filepath):
         cherrypy.response.status = '200 OK'
         return {'message': 'File Moved Successfully'}
     # pylint: enable=invalid-name
+
+    # pylint: disable=invalid-name
+    def DELETE(self, filepath):
+        """Delete a file from WSGI request.
+
+        Delete the file specified in the request to disk.
+        """
+        archivefile = self._archive.open(filepath, 'r')
+        archivefile.close()
+        archivefile.remove()
+        cherrypy.response.status = '200 OK'
+    # pylint: enable=invalid-name
diff --git a/post_deployment_tests/deployment_test.py b/post_deployment_tests/deployment_test.py
@@ -73,9 +73,19 @@ def test_simple_read(self):
         # the len of string 'Writing content for first file'
         self.assertEqual(len(data), 30)
 
+    def test_simple_delete(self):
+        """test reading a simple text file."""
+        fileid = '1238'
+        data = unistr2binary('Writing content for first file')
+        resp = requests.put('{}/{}'.format(ARCHIVEURL, fileid), data=data)
+        self.assertEqual(resp.status_code, 201)
+        resp = requests.delete('{}/{}'.format(ARCHIVEURL, fileid), stream=True)
+        # the len of string 'Writing content for first file'
+        self.assertEqual(resp.status_code, 200)
+
     def test_file_rewrite(self):
         """Test trying to rewrite a file, rewrite should fail."""
-        fileid = '1238'
+        fileid = '1239'
         data = unistr2binary('Writing content for first file')
         resp = requests.put('{}/{}'.format(ARCHIVEURL, fileid), data=data)
         self.assertEqual(resp.status_code, 201)

diff --git a/setup.py b/setup.py
@@ -54,7 +54,10 @@
     packages=find_packages(),
     namespace_packages=['pacifica'],
     entry_points={
-        'console_scripts': ['pacifica-archiveinterface=pacifica.archiveinterface.__main__:main'],
+        'console_scripts': [
+            'pacifica-archiveinterface=pacifica.archiveinterface.__main__:main',
+            'pacifica-archiveinterface-cmd=pacifica.archiveinterface.__main__:cmd'
+        ],
     },
     install_requires=[str(ir.req) for ir in INSTALL_REQS],
     ext_modules=EXT_MODULES

diff --git a/tests/admin_cmd_test.py b/tests/admin_cmd_test.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""File used to unit test the pacifica archive interface."""
+from os.path import sep
+import unittest
+import mock
+from pacifica.archiveinterface.__main__ import cmd
+from .common_setup_test import SetupTearDown
+
+
+class TestPosixBackendArchive(unittest.TestCase, SetupTearDown):
+    """Test the Posix backend archive."""
+
+    @mock.patch('os.unlink')
+    @mock.patch('os.chmod')
+    def test_admin_cmd(self, mock_chmod, mock_unlink):
+        """Test admin command."""
+        temp_fd = open('{}tmp{}1234'.format(sep, sep), 'w')
+        temp_fd.close()
+        del temp_fd
+        mock_unlink.return_value = 0
+        mock_chmod.return_value = 0
+        res = cmd(['delete', '1234'])
+        mock_unlink.assert_called_with('{}tmp{}1234'.format(sep, sep))
+        mock_chmod.assert_called_with('{}tmp{}1234'.format(sep, sep), 0o200)
+        self.assertEqual(res, 0)
diff --git a/tests/cherrypy_test.py b/tests/cherrypy_test.py
@@ -8,9 +8,10 @@
 from pacifica.archiveinterface.archive_utils import bytes_type
 from pacifica.archiveinterface.rest_generator import ArchiveInterfaceGenerator, error_page_default
 from pacifica.archiveinterface.backends.factory import ArchiveBackendFactory
+from .common_setup_test import SetupTearDown
 
 
-class ArchiveInterfaceCPTest(helper.CPWebCase):
+class ArchiveInterfaceCPTest(helper.CPWebCase, SetupTearDown):
     """Base class for all testing classes."""
 
     HOST = '127.0.0.1'
@@ -43,6 +44,8 @@ def test_interface(self):
         self.assertEqual(resp.status_code, 204)
         resp = requests.post('{}/1234'.format(self.url))
         self.assertEqual(resp.status_code, 200)
+        resp = requests.delete('{}/1234'.format(self.url))
+        self.assertEqual(resp.status_code, 200)
 
         with open('/tmp/cptests/222', 'w') as test_fd:
             test_fd.write('this is file 222')
@@ -59,3 +62,7 @@ def test_error_interface(self):
         self.assertEqual(resp.status_code, 500)
         self.assertTrue(
             'No such file or directory' in resp.json()['traceback'])
+        resp = requests.delete('{}/12345'.format(self.url), stream=True)
+        self.assertEqual(resp.status_code, 500)
+        self.assertTrue(
+            'No such file or directory' in resp.json()['traceback'])
diff --git a/tests/common_setup_test.py b/tests/common_setup_test.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Setup and tear down the class cleaning up files."""
+import stat
+from os import remove, chmod, rmdir
+from os.path import isdir, isfile, join, sep
+
+
+class SetupTearDown(object):
+    """Clean up temporary files before and after tests."""
+
+    @staticmethod
+    def _cleanup():
+        """Clean up temporary files in /tmp."""
+        file_list = [
+            '{}{}'.format(sep, join('tmp', '1234')),
+            '{}{}'.format(sep, join('tmp', '1235')),
+            '{}{}'.format(sep, join('tmp', 'cptests', '1235')),
+            '{}{}'.format(sep, join('tmp', '12345')),
+            '{}{}'.format(sep, join('tmp', '5678')),
+            '{}{}'.format(sep, join('tmp', '15', 'cd', '5b', '75bcd15')),
+            '{}{}'.format(sep, join('tmp', 'a', 'b', 'd'))
+        ]
+        dir_list = [
+            '{}{}'.format(sep, join('tmp', 'cptests')),
+            '{}{}'.format(sep, join('tmp', 'a', 'b')),
+            '{}{}'.format(sep, join('tmp', 'a')),
+            '{}{}'.format(sep, join('tmp', '15', 'cd', '5b')),
+            '{}{}'.format(sep, join('tmp', '15', 'cd')),
+            '{}{}'.format(sep, join('tmp', '15')),
+            '{}{}'.format(sep, join('tmp', '39')),
+        ]
+        for fname in file_list:
+            if isfile(fname):
+                chmod(fname, stat.S_IWRITE)
+                remove(fname)
+        for dname in dir_list:
+            if isdir(dname):
+                chmod(dname, stat.S_IWRITE)
+                rmdir(dname)
+
+    def setup_method(self, *_args, **_kwargs):
+        """Call cleanup."""
+        self._cleanup()
+
+    def teardown_method(self, *_args, **_kwargs):
+        """Call cleanup."""
+        self._cleanup()
diff --git a/tests/extended_test.py b/tests/extended_test.py
@@ -5,9 +5,10 @@
 import os
 from pacifica.archiveinterface.backends.posix.extendedfile import extended_file_factory
 from pacifica.archiveinterface.backends.posix.status import PosixStatus
+from .common_setup_test import SetupTearDown
 
 
-class TestExtendedFile(unittest.TestCase):
+class TestExtendedFile(unittest.TestCase, SetupTearDown):
     """Test the ExtendedFile Class."""
 
     def test_posix_file_status(self):
@@ -33,7 +34,7 @@ def test_posix_file_stage(self):
         my_file.close()
 
 
-class TestPosixStatus(unittest.TestCase):
+class TestPosixStatus(unittest.TestCase, SetupTearDown):
     """Test the POSIXStatus Class."""
 
     def test_posix_status_object(self):

diff --git a/tests/interface_test.py b/tests/interface_test.py
@@ -9,9 +9,10 @@
 from pacifica.archiveinterface.exception import ArchiveInterfaceError
 from pacifica.archiveinterface.rest_generator import ArchiveInterfaceGenerator
 from pacifica.archiveinterface.backends.factory import ArchiveBackendFactory
+from .common_setup_test import SetupTearDown
 
 
-class TestArchiveUtils(unittest.TestCase):
+class TestArchiveUtils(unittest.TestCase, SetupTearDown):
     """Test the Archive utils class."""
 
     def test_utils_absolute_path(self):
@@ -50,7 +51,7 @@ def test_get_http_modified_time(self):
             self.assertTrue(hit_exception)
 
 
-class TestId2Filename(unittest.TestCase):
+class TestId2Filename(unittest.TestCase, SetupTearDown):
     """Test the id2filename method."""
 
     def test_id2filename_basic(self):
@@ -89,7 +90,7 @@ def test_id2filename_o_shift_point(self):
         self.assertEqual(filename, '/01/8001')
 
 
-class TestBackendArchive(unittest.TestCase):
+class TestBackendArchive(unittest.TestCase, SetupTearDown):
     """Test the backend archive."""
 
     def test_posix_backend(self):
@@ -108,7 +109,7 @@ def test_invalid_backend(self):
             factory.get_backend_archive('badbackend', '/tmp')
 
 
-class TestArchiveGenerator(unittest.TestCase):
+class TestArchiveGenerator(unittest.TestCase, SetupTearDown):
     """Test the archive generator."""
 
     @staticmethod