Implemented loading external configuration files #67

MANIFEST.in

@@ -11,3 +11,4 @@ include clam/tests/*.py
 include clam/wrappers/*.py
 include clam/wrappers/*.sh
 include clam/config/*.wsgi
+include clam/config/*.yml

clam/common/data.py

@@ -24,6 +24,7 @@
 import io
 import json
 import time
+import yaml
 from copy import copy
 from lxml import etree as ElementTree
 if sys.version < '3':
@@ -171,6 +172,10 @@ class AuthenticationRequired(Exception):
     """This Exception is raised when authentication is required but has not been provided"""
     pass
 
+class ConfigurationError(Exception):
+    """This Exception is raised when authentication is required but has not been provided"""
+    pass
+
 class CLAMFile:
     basedir = ''
 
@@ -2433,5 +2438,50 @@ def unescapeshelloperators(s):
         s = s.replace('%IN%','<')
     return s
 
+def loadconfig(callername, required=True):
+    try:
+        settingsmodule = sys.modules[callername]
+    except:
+        raise ConfigurationError("Unable to determine caller module, got unknown name: " + callername)
+    hostname = os.uname()[1]
+    searchpath = []
+    searchfile = []
+    searchpath.append(os.getcwd())
+    d = os.path.dirname(settingsmodule.__file__)
+    if d:
+        searchpath.append(d)
+    searchfile.append(hostname+'.yml')
+    searchfile.append(hostname+'.yaml')
+    searchfile.append('config.yml')
+    searchfile.append('config.yaml')
+    configfile = None
+
+    #search host-specific configuration
+    if 'CONFIGFILE' in os.environ:
+        if os.environ['CONFIGFILE'][0] == '/' or os.path.exists(os.environ['CONFIGFILE']):
+            configfile = os.environ['CONFIGFILE']
+        else:
+            searchfile.insert(0, os.path.basename(os.environ['CONFIGFILE']))
+
+    if not configfile:
+        for filename in searchfile:
+            for path in searchpath:
+                if os.path.exists(os.path.join(path, filename)):
+                    configfile = os.path.join(path, filename)
+                    break
+            if configfile: break
+    if configfile:
+        with io.open(configfile,'r', encoding='utf-8') as f:
+            data = yaml.safe_load(f.read())
+        for key, value in data.items():
+            setattr(settingsmodule,key.upper(), value)
+        return True
+
+    if required:
+        raise ConfigurationError("Unable to load required external configuration file. Do you need to set the CONFIGFILE environment variable? Detected hostname is " + hostname + "; search path is " + ", ".join(searchpath) + "; searched for " + ", ".join(searchfile))
+    else:
+        return False
+
+
 #yes, this is deliberately placed at the end!
 import clam.common.formats #pylint: disable=wrong-import-position

clam/config/template.py

@@ -50,11 +50,12 @@
 
 # ======== LOCATION ===========
 
-#Add a section for your host:
+#Either add a section for your host here, or
+#specify these variables in an external yaml file
+#called $hostname.yaml or config.yaml and use the loadconfig() mechanism:
 
 host = os.uname()[1]
 if host == "yourhostname":
-
     #The root directory for CLAM, all project files, (input & output) and
     #pre-installed corpora will be stored here. Set to an absolute path:
     ROOT = "/tmp/clam.projects/"
@@ -73,7 +74,14 @@
     #Optionally, you can force the full URL CLAM has to use, rather than rely on any autodetected measures:
     #FORCEURL = "http://yourhostname.com"
 else:
-    raise Exception("I don't know where I'm running from! Add a section in the configuration corresponding to this host (" + os.uname()[1]+")")
+    #This invokes the automatic loader, do not change it;
+    #it will try to find a file named $hostname.yaml (or yml), where $hostname
+    #is the auto-detected hostname of this system. Alternatively, it tries a static config.yml .
+    #You can also set an environment variable CONFIGFILE to specify the exact file to load at run-time.
+    #It will look in several paths including the current working directory and the path this settings script is loaded from.
+    #Such an external configuration file simply defines variables that will be imported here. If it fails to find
+    #a configuration file, an exception will be raised.
+    loadconfig(__name__)
 
 
 

clam/config/textstats.py

@@ -58,6 +58,17 @@
 #USERS = { 'username': pwhash('username', SYSTEM_ID, 'secret') } #Using pwhash and plaintext password in code is not secure!!
 
 
+#Instead of specifying variables like we did above, we can include them from an external configuration file automatically
+#This invokes the automatic loader, it will try to find a file named $hostname.yaml (or yml), where $hostname
+#is the auto-detected hostname of this system. Alternatively, it tries a static config.yml .
+#You can also set an environment variable CONFIGFILE to specify the exact file to load at run-time.
+#It will look in several paths including the current working directory and the path this settings script is loaded from.
+#Such an external configuration file simply defines variables that will be imported here. If it fails to find
+#a configuration file, an exception will be raised.
+loadconfig(__name__)
+
+assert TEST_DUMMY == "test"   #a test, you never need/want this in your own configuration
+
 #Amount of free memory required prior to starting a new process (in MB!), Free Memory + Cached (without swap!)
 #REQUIREMEMORY = 10
 

clam/tests/config.yml

@@ -0,0 +1,2 @@
+TEST_DUMMY: "test"
+