Fix/238 deleteapi

cif/httpd/views/indicators.py

@@ -92,3 +92,26 @@ def post(self):
 
         return jsonify_success(r, code=201)
 
+    def delete(self):
+
+        try:
+            data = request.data.decode('utf-8')
+            logger.debug(data)
+            r = Client(remote, pull_token()).indicators_delete(data)
+
+        except RuntimeError as e:
+            logger.error(e)
+            return jsonify_unknown(msg='submission failed, check logs for more information', code=422)
+
+        except TimeoutError as e:
+            logger.error(e)
+            return jsonify_unknown('submission failed, check logs for more information', 408)
+
+        except Exception as e:
+            logger.error(e)
+            return jsonify_unknown('submission failed, check logs for more information', 422)
+
+        except AuthError:
+            return jsonify_unauth()
+
+        return jsonify_success(r)

cif/router.py

@@ -24,7 +24,7 @@
 HUNTER_ADVANCED = os.getenv('CIF_HUNTER_ADVANCED', 0)
 GATHERER_THREADS = os.getenv('CIF_GATHERER_THREADS', 2)
 STORE_DEFAULT = 'sqlite'
-STORE_PLUGINS = ['cif.store.dummy', 'cif.store.sqlite', 'cif.store.elasticsearch', 'cif.store.rdflib']
+STORE_PLUGINS = ['cif.store.dummy', 'cif.store.sqlite', 'cif.store.elasticsearch']
 
 ZMQ_HWM = 1000000
 ZMQ_SNDTIMEO = 5000

cif/store/__init__.py

@@ -202,6 +202,10 @@ def _flush_create_queue(self):
             if rv['status'] == 'success':
                 self.store.tokens.update_last_activity_at(t, arrow.utcnow().datetime)
 
+    def handle_indicators_delete(self, token, data=None, id=None, client_id=None):
+        t = self.store.tokens.admin(token)
+        return self.store.indicators.delete(t, data=data, id=id)
+
     def handle_indicators_create(self, token, data, id=None, client_id=None):
         # this will raise AuthError if false
         t = self.store.tokens.write(token)

cif/store/sqlite/indicator.py

@@ -342,10 +342,10 @@ def _filter_terms(self, filters, s):
             if k == 'reporttime':
                 if ',' in filters[k]:
                     start, end = filters[k].split(',')
-                    s = s.filter(Indicator.reporttime >= start)
-                    s = s.filter(Indicator.reporttime <= end)
+                    s = s.filter(Indicator.reporttime >= arrow.get(start).datetime)
+                    s = s.filter(Indicator.reporttime <= arrow.get(end).datettime)
                 else:
-                    s = s.filter(Indicator.reporttime >= filters[k])
+                    s = s.filter(Indicator.reporttime >= arrow.get(filters[k]).datetime)
 
             elif k == 'reporttimeend':
                 s = s.filter(Indicator.reporttime <= filters[k])
@@ -395,7 +395,7 @@ def _filter_groups(self, token, s):
         s = s.filter(or_(Indicator.group == g for g in groups))
         return s
 
-    def search(self, token, filters, limit=500):
+    def _search(self, filters, token):
         logger.debug('running search')
 
         myfilters = dict(filters.items())
@@ -407,6 +407,10 @@ def search(self, token, filters, limit=500):
         s = self._filter_indicator(myfilters, s)
         s = self._filter_terms(myfilters, s)
         s = self._filter_groups(token, s)
+        return s
+
+    def search(self, token, filters, limit=500):
+        s = self._search(filters, token)
 
         limit = filters.pop('limit', limit)
 
@@ -418,6 +422,31 @@ def search(self, token, filters, limit=500):
 
         logger.debug('done: %0.4f' % (time.time() - start))
 
+    def delete(self, token, data=None, id=None):
+        if type(data) is not list:
+            data = [data]
+
+        ids = []
+        for d in data:
+            if d.get('id'):
+                ids.append(Indicator.id == d['id'])
+                logger.debug('removing: %s' % d['id'])
+            else:
+                ss = self._search(d, token)
+                for i in ss:
+                    ids.append(Indicator.id == i.id)
+                    logger.debug('removing: %s' % i.indicator)
+
+        if len(ids) == 0:
+            return 0
+
+        s = self.handle().query(Indicator)
+        s = s.filter(or_(*ids))
+        rv = s.delete()
+        self.handle().commit()
+
+        return rv
+
     def upsert(self, token, data):
         if type(data) == dict:
             data = [data]

test/zsqlite/test_store_sqlite_indicators.py

@@ -157,3 +157,24 @@ def test_store_sqlite_indicators(store, indicator):
 
     assert x is None
 
+    r = store.handle_indicators_delete(t, data=[{
+        'indicator': 'example.com',
+    }])
+    assert r == 2
+
+    x = store.handle_indicators_search(t, {
+        'indicator': 'example.com',
+        'nolog': 1
+    })
+    assert len(x) == 0
+
+    x = store.handle_indicators_search(t, {
+        'indicator': 'example2.com',
+        'nolog': 1
+    })
+
+    for xx in x:
+        r = store.handle_indicators_delete(t, data=[{
+            'id': xx['id']
+        }])
+        assert r == 1