Fix #100 Add Better Exclude (#105)

* Fix pre-commit issues

Signed-off-by: David Brown <dmlb2000@gmail.com>

* Add exclude to test right

Signed-off-by: David Brown <dmlb2000@gmail.com>

* Add docs for readme

Signed-off-by: David Brown <dmlb2000@gmail.com>

* list not dict

Signed-off-by: David Brown <dmlb2000@gmail.com>

* Try doing a type cast on the exclude value

Signed-off-by: David Brown <dmlb2000@gmail.com>

* Try a different way to specify the exclude

Signed-off-by: David Brown <dmlb2000@gmail.com>

* Add extra exception to continue block

Signed-off-by: David Brown <dmlb2000@gmail.com>

* put more into the try block

Signed-off-by: David Brown <dmlb2000@gmail.com>

docs/exampleusage.md

@@ -224,5 +224,5 @@ optional arguments:
 Example command lines from the test suite.
 
 ```
-pacifica-policy-cmd searchsync --objects-per-page=4 --threads=1 --time-ago='7 days ago' --exclude='keys.104'
+pacifica-policy-cmd searchsync --objects-per-page=4 --threads=1 --time-ago='7 days ago' --exclude='keys._id=104'
 ```

pacifica/policy/admin_cmd.py

@@ -67,12 +67,19 @@ def create_subcommands(subparsers):
     return datarel_parser, searchsync_parser
 
 
+def exclude_options(obj_str):
+    """Turn an object string into expressive exclude option."""
+    obj_cls_attr, value = obj_str.split('=')
+    obj_cls, obj_attr = obj_cls_attr.split('.')
+    return (obj_cls, obj_attr, value)
+
+
 def searchsync_options(searchsync_parser):
     """Add the searchsync command line options."""
     searchsync_parser.add_argument(
         '--exclude', dest='exclude',
-        help='type.id of object to exclude (i.e. --exclude="projects.1234").',
-        nargs='*', default=set(), type=text_type
+        help='object and attr to exclude (i.e. --exclude="projects._id=1234").',
+        nargs='*', default=set(), type=exclude_options
     )
     searchsync_parser.add_argument(
         '--objects-per-page', default=40000,

pacifica/policy/search_render.py

@@ -18,12 +18,25 @@ def get_render_class(obj_cls):
         parts = [obj_cls_part.capitalize() for obj_cls_part in obj_cls.split('_')]
         return getattr(obj_mod, '{}Render'.format(''.join(parts)))
 
+    @classmethod
+    def object_exclude(cls, obj_cls, obj, exclude):
+        """Check if object is part of the exclude tuples."""
+        for xobj_cls, xobj_attr, xvalue in exclude:
+            try:
+                xattr_type = type(obj[xobj_attr])
+                typed_xvalue = xattr_type(xvalue)
+            except (ValueError, KeyError):
+                continue
+            if xobj_cls == obj_cls and xobj_attr in obj and obj[xobj_attr] == typed_xvalue:
+                return True
+        return False
+
     @classmethod
     def generate(cls, obj_cls, objs, exclude):
         """generate the institution object."""
         render_cls = cls.get_render_class(obj_cls)
         for obj in objs:
-            if u'{}.{}'.format(obj_cls, obj.get('_id', '')) in exclude:
+            if cls.object_exclude(obj_cls, obj, exclude):
                 continue
             yield {
                 '_op_type': 'update',

tests/admin_cmd_test.py

@@ -157,7 +157,7 @@ class TestAdminCMD(TestCase):
 
     def test_default_search_sync(self):
         """Test the data release subcommand."""
-        main('searchsync', '--objects-per-page', '4', '--threads', '1', '--exclude', 'keys.104')
+        main('searchsync', '--objects-per-page', '4', '--threads', '1', '--exclude', 'keys.key=temp_f')
         resp = requests.get('http://localhost:9200/pacifica_search/_stats')
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(resp.json()['indices']['pacifica_search']['primaries']['docs']['count'], 44)