security/q-feeds-connector: sync with master

Author: fichtner

security/q-feeds-connector/Makefile

@@ -1,6 +1,6 @@
 PLUGIN_NAME=		q-feeds-connector
 PLUGIN_VERSION=		1.5
-PLUGIN_REVISION=	2
+PLUGIN_REVISION=	3
 PLUGIN_COMMENT=		Connector for Q-Feeds threat intel
 PLUGIN_MAINTAINER=	devel@qfeeds.com
 PLUGIN_TIER=		2

security/q-feeds-connector/pkg-descr

@@ -11,6 +11,9 @@ Plugin Changelog
 * Feature: Add dest address for unbound
 * Bugfix: Invalidate alias cache on reconfigure
 * Bugfix: Ignore "pass" log lines for `qfeedsctl.py logs`
+* Bugfix: Use proper configctl command option for flushing
+* Bugfix: Add optional locked mode in qfeedsctl.py for cron runners and wait for configfile changes when HTTP 401 is thrown
+* Bugfix: Ignore invalid json index file leading to instant exit of qfeedsctl.py
 
 1.4
 

security/q-feeds-connector/src/etc/inc/plugins.inc.d/qfeeds.inc

@@ -28,8 +28,8 @@
 
 function qfeeds_cron()
 {
-    $jobs = [];
-    $jobs[]['autocron'] = ['/usr/local/sbin/configctl -d qfeeds update', '*/5'];
-    $jobs[]['autocron'] = ['/usr/local/sbin/configctl -d ! qfeeds stats', '*/15'];
-    return $jobs;
+    return [
+        ['autocron' => ['/usr/local/sbin/configctl -d qfeeds update', '*/5']],
+        ['autocron' => ['/usr/local/sbin/configctl -df qfeeds stats', '*/15']],
+    ];
 }

security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/__init__.py

@@ -63,7 +63,10 @@ def index(self):
             list(self.fetch_index())
         elif not os.path.exists(self.index_file):
             return {}
-        data = ujson.load(open(self.index_file)) or {}
+        try:
+            data = ujson.load(open(self.index_file)) or {}
+        except ujson.JSONDecodeError:
+            data = {}
         if type(data) is dict:
             for feed in data.get('feeds', []):
                 feed['local_filename'] = "%s/%s.txt" % (self._target_dir, feed['feed_type'])

security/q-feeds-connector/src/opnsense/scripts/qfeeds/lib/api.py

@@ -23,22 +23,31 @@
     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     POSSIBILITY OF SUCH DAMAGE.
 """
+
 import os
 import requests
 from configparser import ConfigParser
 
 
 class QFeedsConfig:
+    config_filename = '/usr/local/etc/qfeeds.conf'
+    conf_timestamp = None
     api_key = None
 
     def __init__(self):
-        config_filename = '/usr/local/etc/qfeeds.conf'
-        if os.path.isfile(config_filename):
+        if os.path.isfile(self.config_filename):
             cnf = ConfigParser()
-            cnf.read(config_filename)
+            cnf.read(self.config_filename)
             if cnf.has_section('api') and cnf.has_option('api', 'key'):
                 self.api_key = cnf.get('api', 'key').strip()
 
+            if self.conf_timestamp is None:
+                QFeedsConfig.conf_timestamp = os.stat(self.config_filename).st_mtime
+
+    @classmethod
+    def has_changed(cls):
+        return os.path.isfile(cls.config_filename) and cls.conf_timestamp != os.stat(cls.config_filename).st_mtime
+
 
 class Api:
     def __init__(self):

security/q-feeds-connector/src/opnsense/scripts/qfeeds/qfeedsctl.py

@@ -27,19 +27,34 @@
 """
 
 import argparse
+import fcntl
+import time
 import sys
 import ujson
 from requests.exceptions import HTTPError, Timeout
 from lib import QFeedsActions
+from lib.api import QFeedsConfig
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--target_dir', default='/var/db/qfeeds-tables')
     parser.add_argument('-f', help='forced (auto index)' , default=False, action='store_true')
     parser.add_argument('-v', help='verbose output' , default=False, action='store_true')
+    parser.add_argument('-l', help='lock operation' , default=False, action='store_true')
     parser.add_argument("action", choices=QFeedsActions.list_actions(), nargs='*')
     args = parser.parse_args()
+
+    fhandle = None
+    if args.l:
+        lck_filename = '/tmp/qfeeds_prc.LCK'
+        fhandle = open(lck_filename, 'a+')
+        try:
+            fcntl.flock(fhandle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError:
+            print('already busy, exit')
+            sys.exit(0)
+
     if args.v:
         # verbose mode
         import http.client as http_client
@@ -51,6 +66,14 @@
                 print(msg)
     except HTTPError as exc:
         print('exit with HTTPError %d (%s)' % (exc.response.status_code, exc.response.text))
+        if exc.response.status_code == 401 and 'update' in args.action:
+            print('batch mode - wait for configuration update or timeout')
+            t_start = time.time()
+            while not QFeedsConfig.has_changed():
+                if time.time() - t_start > 3600:
+                    print('timeout waiting for config change')
+                    break
+                time.sleep(5)
         sys.exit(-1)
     except Timeout as exc:
         print('timeout reaching api endpoint')
@@ -61,3 +84,6 @@
     except ujson.JSONDecodeError:
         print("JSON decode error")
         sys.exit(-1)
+    finally:
+        if fhandle:
+            fcntl.flock(fhandle, fcntl.LOCK_UN)

security/q-feeds-connector/src/opnsense/service/conf/actions.d/actions_qfeeds.conf

@@ -6,7 +6,7 @@ message:reconfigure QFeeds
 errors:no
 
 [update]
-command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py update
+command:/usr/local/opnsense/scripts/qfeeds/qfeedsctl.py -l update
 parameters:
 type:script_output
 message:update QFeeds