Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: neonknight/postomaat
base: 3cad99c04b
...
head fork: neonknight/postomaat
compare: 16a787d4f7
  • 14 commits
  • 13 files changed
  • 0 commit comments
  • 3 contributors
View
11 README.md
@@ -1,4 +1,13 @@
postomaat
=========
-A policy daemon for postfix written in python
+A policy daemon for postfix written in python
+
+This is a by-product of the [Fuglu](https://github.com/gryphius/fuglu) mailfilter.
+While fuglu focuses on scanning a full message (pre- or after-queue), postomaat only uses the message
+fields available in the [Postfix policy access delegation protocol](http://www.postfix.org/SMTPD_POLICY_README.html)
+It can therefore make decisions much faster than fuglu, but only based on envelope data (sender adress, recipient adress, client ip etc).
+Postomaat can not make decisions based on message headers or body.
+
+Warning: Postomaat currently doesn't receive the same testing as fuglu before commiting to github.
+The master branch therefore might or might not work out of the box.
View
2  conf/conf.d/dbwriter.conf.dist
@@ -1,4 +1,4 @@
[DBWriter]
-dbconnection=mysql://root@localhost?charset=utf8
+dbconnection=mysql://root@localhost/dbwriter?charset=utf8
table=maillog
fields=from_address to_address from_domain to_domain size queue_id:queueid recipient_count:num_recipients client_address:ip client_name:revdns helo_name:helo sasl_sender:sasl_user sasl_method
View
5 conf/conf.d/geoip.conf.dist
@@ -1,6 +1,9 @@
[GeoIP]
+# location of the MaxMind GeopIP database file
database=/data/geoip/GeoIP.dat
+# list of countries you do not want to receive mail from.
blacklist=CN,IN,KP,KR,TW,VN,A1,A2
+# list of countries you want want to receive mail from. all other countries will be rejected. If you specify a whitelist, the blacklist will have no function.
whitelist=
-#what to do with unknown countries? this affects local IP-addresses. Set this to DUNNO or REJECT
+# what to do with unknown countries? this affects local IP-addresses. Set this to DUNNO or REJECT
on_unknown=DUNNO
View
2  conf/postomaat.conf.dist
@@ -14,7 +14,7 @@ user=nobody
group=nobody
#what plugins do we load, comma separated
-plugins=postomaat.plugins.call-ahead.AddressCheck
+plugins=
#custom plugin dir
plugindir=
View
6 doc/TODO
@@ -1,4 +1,6 @@
- documentation
- alias system
-- more default plugins (greylist?)
-- unicode bug in Call-Ahead
+- more default plugins (greylist, quotacheck, ..)
+- unicode bug in Call-Ahead still around?
+- apply_template function from fuglu
+- postomaat_conf like fuglu_conf
View
8 src/postomaat/core.py
@@ -307,8 +307,11 @@ def test(self,valuedict):
suspect=Suspect(valuedict)
if not self.load_plugins():
sys.exit(1)
- result=SessionHandler(None, self.config, self.plugins).run_plugins(suspect, self.plugins)
- return result
+ sesshandler=SessionHandler(None, self.config, self.plugins)
+ sesshandler.run_plugins(suspect, self.plugins)
+ action=sesshandler.action
+ arg=sesshandler.arg
+ return (action,arg)
def shutdown(self):
for server in self.servers:
@@ -417,6 +420,7 @@ def load_plugins(self):
if allOK:
self.plugins=newplugins
+ self.propagate_plugin_defaults()
return allOK
View
17 src/postomaat/db.py
@@ -1,10 +1,14 @@
-from sqlalchemy import create_engine
-from sqlalchemy.orm import scoped_session, sessionmaker
+SQLALCHEMY_AVAILABLE=False
+try:
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import scoped_session, sessionmaker
+ SQLALCHEMY_AVAILABLE=True
+except:
+ pass
_enginecache={}
-
-def get_connection(dburl):
+def get_session(dburl):
if dburl in _enginecache:
engine= _enginecache[dburl]
else:
@@ -14,4 +18,7 @@ def get_connection(dburl):
session = scoped_session(maker)
session.configure(bind=engine)
return session
-
+
+
+def get_gonnection(dburl):
+ return get_session(dburl)
View
58 src/postomaat/plugins/call-ahead.py
@@ -1,16 +1,12 @@
#!/usr/bin/python
import sys
-#TODO: full rewrite to sqlalchemy
-#TODO: propagate tables
-
-
#in case the tool is not installed system wide (development...)
if __name__ =='__main__':
sys.path.append('../../')
from postomaat.shared import *
-from postomaat.db import *
+from postomaat.db import SQLALCHEMY_AVAILABLE,get_session
from threading import Lock
import time
import smtplib
@@ -50,9 +46,31 @@ def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.cache=MySQLCache(config)
- self.requiredvars=((self.section,'dbconnection'),(self.section,'always_assume_rec_verification_support'),(self.section,'always_accept'))
+ self.requiredvars={
+ 'dbconnection':{
+ 'default':"mysql://root@localhost/callahead?charset=utf8",
+ 'description':'SQLAlchemy Connection string',
+ },
+
+ 'always_assume_rec_verification_support':{
+ 'default': "False",
+ 'description': """set this to true to disable the blacklisting of servers that don't support recipient verification"""
+
+ },
+
+ 'always_accept':{
+ 'default': "False",
+ 'description': """Set this to always return 'DUNNO' but still perform the recipient check and fill the cache (learning mode without rejects)"""
+
+ },
+
+ }
def lint(self):
+ if not SQLALCHEMY_AVAILABLE:
+ print "sqlalchemy is not installed"
+ return False
+
if not self.checkConfig():
return False
try:
@@ -303,7 +321,7 @@ def get_relays(self,domain,domainconfig=None):
(tp,val)=serverconfig.split(':',1)
if tp=='sql':
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
ret=conn.execute(val)
return [result[0] for result in ret]
elif tp=='mx':
@@ -443,7 +461,7 @@ def __init__(self,config):
def blacklist(self,domain,relay,seconds,failstage='rcpt_to',reason='unknown'):
"""Put a domain/relay combination on the recipient verification blacklist for a certain amount of time"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
statement="""INSERT INTO ca_blacklist (domain,relay,expiry_ts,check_stage,reason) VALUES (:domain,:relay,now()+interval :interval second,:checkstag,:reason)
ON DUPLICATE KEY UPDATE expiry_ts=now()+interval :interval second,check_stage=:checkstage,reason=:reason
@@ -459,7 +477,7 @@ def blacklist(self,domain,relay,seconds,failstage='rcpt_to',reason='unknown'):
def is_blacklisted(self,domain,relay):
"""Returns True if the server/relay combination is currently blacklisted and should not be used for recipient verification"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return False
statement="SELECT reason FROM ca_blacklist WHERE domain=:domain and relay=:relay and expiry_ts>now()"
@@ -469,7 +487,7 @@ def is_blacklisted(self,domain,relay):
def unblacklist(self,relayordomain):
"""remove a server from the blacklist/history"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
statement="""DELETE FROM ca_blacklist WHERE domain=:removeme or relay=:removeme"""
values={'removeme':relayordomain}
res=conn.execute(statement,values)
@@ -477,7 +495,7 @@ def unblacklist(self,relayordomain):
def get_blacklist(self):
"""return all blacklisted servers"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return None
statement="SELECT domain,relay,reason,expiry_ts FROM ca_blacklist WHERE expiry_ts>now() ORDER BY domain"
@@ -486,7 +504,7 @@ def get_blacklist(self):
return result
def wipe_address(self,address):
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return
@@ -496,7 +514,7 @@ def wipe_address(self,address):
return res.rowcount
def cleanup(self):
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
postime=self.config.getint('AddressCheck','keep_positive_history_time')
negtime=self.config.getint('AddressCheck','keep_negative_history_time')
statement="""DELETE FROM ca_addresscache WHERE positive=:pos and expiry_ts<(now() -interval :keeptime day)"""
@@ -517,7 +535,7 @@ def wipe_domain(self,domain,positive=None):
if positive is False all negative cache entries are deleted
if positive is True, all positive cache entries are deleted
"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return
@@ -534,7 +552,7 @@ def wipe_domain(self,domain,positive=None):
def put_address(self,address,seconds,positiveEntry=True,message=None):
"""put address into the cache"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return
statement="""INSERT INTO ca_addresscache (email,domain,expiry_ts,positive,message) VALUES (:email,:domain,now()+interval :interval second,:positive,:message)
@@ -553,7 +571,7 @@ def put_address(self,address,seconds,positiveEntry=True,message=None):
def get_address(self,address):
"""Returns a tuple (positive(boolean),message) if a cache entry exists, None otherwise"""
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return
statement="SELECT positive,message FROM ca_addresscache WHERE email=:email and expiry_ts>now()"
@@ -562,7 +580,7 @@ def get_address(self,address):
return res.first()
def get_all_addresses(self,domain):
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
if not conn:
return None
statement="SELECT email,positive FROM ca_addresscache WHERE domain=:domain and expiry_ts>now() ORDER BY email"
@@ -572,7 +590,7 @@ def get_all_addresses(self,domain):
return result
def get_total_counts(self):
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
statement="SELECT count(*) FROM ca_addresscache WHERE expiry_ts>now() and positive=1"
result=conn.execute(statement)
poscount=result.fetchone()[0]
@@ -605,14 +623,14 @@ def __init__(self,config):
def get_domain_config_value(self,domain,key):
retval=None
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
res=conn.execute("SELECT confvalue FROM ca_configoverride WHERE domain=:domain and confkey=:confkey",{'domain':domain,'confkey':key})
return res.scalar()
def get_domain_config_all(self,domain):
retval=dict()
- conn=get_connection(self.config.get('AddressCheck','dbconnection'))
+ conn=get_session(self.config.get('AddressCheck','dbconnection'))
res=conn.execute("SELECT confkey,confvalue FROM ca_configoverride WHERE domain=:domain",{'domain':domain})
for row in res:
retval[row[0]]=row[1]
View
30 src/postomaat/plugins/dbwriter.py
@@ -1,11 +1,30 @@
from postomaat.shared import *
-from postomaat.db import *
+from postomaat.db import SQLALCHEMY_AVAILABLE,get_session
+
class DBWriter(ScannerPlugin):
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
+ self.requiredvars={
+ 'dbconnection':{
+ 'default':"mysql://root@localhost/dbwriter?charset=utf8",
+ 'description':'SQLAlchemy Connection string',
+ },
+
+ 'table':{
+ 'default': "maillog",
+ 'description': """Tablename where we should insert mails"""
+
+ },
+
+ 'fields':{
+ 'default': "from_address to_address from_domain to_domain size queue_id:queueid",
+ 'description': """Fields that should be inserted. use <fieldname>:<columnname> or just <fieldname> if the database column name matches the fieldname"""
+ },
+
+ }
def get_fieldmap(self):
"""create the mapping from tags to column names based on the config string
@@ -30,6 +49,11 @@ def get_fieldmap(self):
return fieldmap
def lint(self):
+ if not SQLALCHEMY_AVAILABLE:
+ print "sqlalchemy is not installed"
+ return False
+
+
#check fieldmap, select all fields (if we can't select, we can't insert)
if not self.checkConfig():
return False
@@ -39,7 +63,7 @@ def lint(self):
requiredcolumnnames=fieldmap.keys()
dbcolumns=",".join(requiredcolumnnames)
try:
- conn=get_connection(self.config.get(self.section,'dbconnection'))
+ conn=get_session(self.config.get(self.section,'dbconnection'))
except Exception,e:
print "DB Connection failed. Reason: %s"%(str(e))
return False
@@ -104,7 +128,7 @@ def examine(self,suspect):
#print sql_insert
#print data
- conn=get_connection(self.config.get(self.section,'dbconnection'))
+ conn=get_session(self.config.get(self.section,'dbconnection'))
conn.execute(sql_insert,data)
except Exception,e:
self.logger.error("DB Writer plugin failed, Log not written. : %s"%str(e))
View
18 src/postomaat/plugins/geoip.py
@@ -80,8 +80,7 @@ def _loadData(self, filename):
class GeoIPCache(FuFileCache):
def _initlocal(self, **kw):
- if not os.path.exists(self.file):
- raise IOError('Could not find GeoIP database %s' % self.file)
+ self.geoip = None
def _reallyloadData(self, filename):
@@ -111,6 +110,7 @@ class GeoIPPlugin(ScannerPlugin):
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
+ self.geoip = None
@@ -118,6 +118,12 @@ def examine(self,suspect):
if not have_geoip:
return DUNNO
+ database = self.config.get('GeoIP', 'database')
+ if not os.path.exists(database):
+ return DUNNO
+ if not self.geoip:
+ self.geoip = GeoIPCache(database)
+
client_address=suspect.get_value('client_address')
if client_address is None:
self.logger.error('No client address found')
@@ -132,9 +138,6 @@ def examine(self,suspect):
if on_unknown.strip().upper() == 'REJECT':
unknown = REJECT
- database = self.config.get('GeoIP', 'database')
- self.geoip = GeoIPCache(database)
-
cc = self.geoip.country_code(client_address)
cn = self.geoip.country_name(cc)
@@ -159,6 +162,11 @@ def lint(self):
if not have_geoip:
print 'pygeoip module not installed - this plugin will do nothing'
lint_ok = False
+
+ database = self.config.get('GeoIP', 'database')
+ if not os.path.exists(database):
+ print 'Could not find geoip database file - this plugin will do nothing'
+ lint_ok = False
if not self.checkConfig():
print 'Error checking config'
View
145 src/postomaat/plugins/script.py
@@ -0,0 +1,145 @@
+from postomaat.shared import ScannerPlugin,DUNNO,ACCEPT,DEFER,REJECT
+import os
+import traceback
+import time
+
+class Stopped(Exception):
+ pass
+
+class ScriptFilter(ScannerPlugin):
+ """ This plugins executes scripts found in a specified directory.
+This can be used to quickly add a custom filter script without changing the postomaat configuration.
+
+Filterscripts must be written in standard python but with the file ending ``.pmf`` ("postomaat filter")
+
+scripts are reloaded for every message executed in alphabetic order
+
+The API is basically the same as for normal plugins within the ``examine()`` method, with a few special cases:
+
+there is no 'self' which means:
+
+ * access the configuration by using ``config`` directly (instead of ``self.config``)
+ * use ``debug('hello world')`` instead of ``self.logger.debug('hello world')``
+
+the script should not return anything, but change the available variables ``action`` and ``message`` instead
+(``DUNNO``, ``REJECT``, ``DEFER``, ``ACCEPT`` are already imported)
+
+use ``stop()`` to exit the script
+
+
+example script:
+(put this in /etc/postomaat/scriptfilter/99_demo.pmf for example)
+
+::
+
+ #block all messages from evilsender.example.com
+ #TODO: demo script here
+ action=REJECT
+ message="you shall not pass"
+
+
+ """
+ def __init__(self,config,section=None):
+ ScannerPlugin.__init__(self,config,section)
+ self.logger=self._logger()
+ self.requiredvars={
+ 'scriptdir':{
+ 'default':'/etc/postomaat/scriptfilter',
+ 'description':'Dir that contains the scripts (*.pmf files)',
+ }
+ }
+
+ def examine(self,suspect):
+ starttime=time.time()
+ scripts=self.get_scripts()
+ retaction=DUNNO
+ retmessage=''
+ for script in scripts:
+ self.logger.debug("Executing script %s"%script)
+ sstart=time.time()
+ action,message=self.exec_script(suspect, script)
+ send=time.time()
+ self.logger.debug("Script %s done in %.4fs result: %s %s"%(script,send-sstart,action,message))
+ if action!=DUNNO:
+ retaction=action
+ retmessage=message
+ break
+
+ endtime=time.time()
+ difftime=endtime-starttime
+ suspect.tags['ScriptFilter.time']="%.4f"%difftime
+ return retaction,retmessage
+
+
+ def lint(self):
+ allok=(self.checkConfig() and self.lint_code())
+ return allok
+
+ def lint_code(self):
+ scriptdir=self.config.get(self.section,'scriptdir')
+ if not os.path.isdir(scriptdir):
+ print "Script dir %s does not exist"%scriptdir
+ return False
+ scripts=self.get_scripts()
+ counter=0
+ for script in scripts:
+ counter+=1
+ try:
+ source=open(script,'r').read()
+ compile(source,script,'exec')
+ except:
+ trb=traceback.format_exc()
+ print "Script %s failed to compile: %s"%(script,trb)
+ return False
+ print "%s scripts found"%counter
+ return True
+
+ def _debug(self,suspect,message):
+ self.logger.debug(message)
+
+
+
+ def exec_script(self,suspect,filename):
+ action=DUNNO
+ message=''
+ debug = lambda message: self._debug(suspect,message)
+
+ def stop():
+ raise Stopped()
+
+ scriptlocals=dict(
+ action=action,
+ message=message,
+ suspect=suspect,
+ debug=debug,
+ config=self.config,
+ stop=stop,
+ DUNNO=DUNNO,ACCEPT=ACCEPT,DEFER=DEFER,REJECT=REJECT,
+
+ )
+
+ scriptglobals=globals().copy()
+ try:
+ execfile(filename,scriptglobals,scriptlocals)
+ action=scriptlocals['action']
+ message=scriptlocals['message']
+ except Stopped:
+ pass
+ except:
+ trb=traceback.format_exc()
+ self.logger.error("Script %s failed: %s"%(filename,trb))
+
+ return action,message
+
+ def get_scripts(self):
+ scriptdir=self.config.get(self.section,'scriptdir')
+ if os.path.isdir(scriptdir):
+ filelist=os.listdir(scriptdir)
+ scripts=[os.path.join(scriptdir,f) for f in filelist if f.endswith('.pmf')]
+ scripts=sorted(scripts)
+ return scripts
+ else:
+ return []
+
+ def __str__(self):
+ return "Scriptfilter Plugin"
View
97 src/postomaat/shared.py
@@ -84,7 +84,54 @@ def get_tag(self,key):
return self.tags[key]
def __str__(self):
- return "Suspect: %s"%self.tags
+ return "Suspect:sender=%s recipient=%s tags=%s"%(self.from_address, self.to_address, self.tags)
+
+ @property
+ def from_address(self):
+ sender=self.get_value('sender')
+ if sender==None:
+ return None
+
+ try:
+ addr=strip_address(sender)
+ return addr
+ except:
+ return None
+
+ @property
+ def from_domain(self):
+ from_address=self.from_address
+ if from_address==None:
+ return None
+
+ try:
+ return extract_domain(from_address)
+ except:
+ return None
+
+ @property
+ def to_address(self):
+ rec=self.get_value('recipient')
+ if rec==None:
+ return None
+
+ try:
+ addr=strip_address(rec)
+ return addr
+ except:
+ return None
+
+ @property
+ def to_domain(self):
+ rec=self.to_address
+ if rec==None:
+ return None
+ try:
+ return extract_domain(rec)
+ except:
+ return None
+
+
##it is important that this class explicitly extends from object, or __subclasses__() will not work!
class BasicPlugin(object):
@@ -109,16 +156,44 @@ def lint(self):
def checkConfig(self):
allOK=True
- for configvar in self.requiredvars:
- (section,config)=configvar
- try:
- var=self.config.get(section,config)
- except ConfigParser.NoOptionError:
- print "Missing configuration value [%s] :: %s"%(section,config)
- allOK=False
- except ConfigParser.NoSectionError:
- print "Missing configuration section %s"%(section)
- allOK=False
+
+ #old config style
+ if type(self.requiredvars)==tuple or type(self.requiredvars)==list:
+ for configvar in self.requiredvars:
+ if type(self.requiredvars)==tuple:
+ (section,config)=configvar
+ else:
+ config=configvar
+ section=self.section
+ try:
+ var=self.config.get(section,config)
+ except ConfigParser.NoOptionError:
+ print "Missing configuration value [%s] :: %s"%(section,config)
+ allOK=False
+ except ConfigParser.NoSectionError:
+ print "Missing configuration section %s"%(section)
+ allOK=False
+
+ #new config style
+ if type(self.requiredvars)==dict:
+ for config,infodic in self.requiredvars.iteritems():
+ section=self.section
+ if 'section' in infodic:
+ section=infodic['section']
+
+ try:
+ var=self.config.get(section,config)
+ if 'validator' in infodic:
+ if not infodic["validator"](var):
+ print "Validation failed for [%s] :: %s"%(section,config)
+ allOK=False
+ except ConfigParser.NoSectionError:
+ print "Missing configuration section [%s] :: %s"%(section,config)
+ allOK=False
+ except ConfigParser.NoOptionError:
+ print "Missing configuration value [%s] :: %s"%(section,config)
+ allOK=False
+
return allOK
View
3  src/startscript/postomaat
@@ -202,7 +202,8 @@ elif debugmsg:
else:
print "Warning: %s ignored - unsupported attribute"%key
print attrs
- controller.test(attrs)
+ action,arg=controller.test(attrs)
+ print "Result: %s %s"%(action,arg)
else:
signal.signal(signal.SIGHUP, sighup)
controller.startup()

No commit comments for this range

Something went wrong with that request. Please try again.