Adds filtering to log observers

At the moment, it only lets through log events with a system that starts with 'SwFTP' and '-'
* swftp.logging.StdOutObserver - outputs logging to stdout
* swftp.logging.LOG_USER, swftp.logging.LOG_LOCAL0, etc. - Outputs to log facility

README.md

@@ -209,8 +209,8 @@ Packaged with SwFTP are a set of example init scripts, upstart scripts. They are
     * /etc/init/swftp-ftp.conf
     * /etc/init/swftp-sftp.conf
 * init.d
-    * /etc/init/swftp-ftp
-    * /etc/init/swftp-sftp
+    * /etc/init.d/swftp-ftp
+    * /etc/init.d/swftp-sftp
 * Supervisor
     * /etc/supervisor/conf.d/swftp.conf
 * Example swftp.conf file

swftp/ftp/server.py

@@ -15,6 +15,7 @@
 from twisted.internet.protocol import Protocol
 from twisted.python import log
 
+from swftp.logging import msg
 from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path
 from swftp.swift import NotFound, Conflict
 
@@ -56,6 +57,11 @@ def connectionLost(self, *args, **kwargs):
 
         if self.shell:
             username = self.shell.username()
+            msg("User Disconnected (%s) [%s/%s]" % (
+                username,
+                self._connCountMap[username],
+                self.maxConnectionsPerUser,
+                ))
             self._connCountMap[username] -= 1
             # To avoid a slow memory leak
             if self._connCountMap[username] == 0:
@@ -69,13 +75,18 @@ def ftp_PASS(self, *args, **kwargs):
         def pass_cb(res):
             username = self.shell.username()
             self._connCountMap[username] += 1
+            msg("User Connected (%s) [%s/%s]" % (
+                username,
+                self._connCountMap[username],
+                self.maxConnectionsPerUser,
+                ))
             if self.maxConnectionsPerUser != 0 and \
                     self._connCountMap[username] > self.maxConnectionsPerUser:
-                log.msg("Too Many Connections For User %s [%s/%s]" % (
+                msg("Too Many Connections For User (%s) [%s/%s]" % (
                     username,
                     self._connCountMap[username],
                     self.maxConnectionsPerUser,
-                ))
+                    ))
                 self.sendLine(RESPONSE[TOO_MANY_CONNECTIONS])
                 self.transport.loseConnection()
             return res
@@ -124,9 +135,9 @@ def __init__(self, swiftconn):
 
     def log_command(self, command, *args):
         arg_list = ', '.join(str(arg) for arg in args)
-        log.msg("COMMAND: %s(%s)" % (command, arg_list),
-                system="SwFTP-FTP, (%s)" % self.swiftconn.username,
-                metric='command.%s' % command)
+        msg("cmd: %s(%s)" % (command, arg_list),
+            system="SwFTP-FTP, (%s)" % self.swiftconn.username,
+            metric='command.%s' % command)
 
     def username(self):
         return self.swiftconn.username
@@ -274,7 +285,6 @@ def openForWriting(self, path):
         fullpath = self._fullpath(path)
         container, obj = obj_to_path(fullpath)
         if not container or not obj:
-            log.msg('cannot upload to root')
             raise CmdNotImplementedForArgError(
                 'Cannot upload files to root directory.')
         f = SwiftWriteFile(self.swiftfilesystem, fullpath)

swftp/ftp/service.py

@@ -4,6 +4,7 @@
 See COPYING for license information.
 """
 from swftp import VERSION
+from swftp.logging import StdOutObserver
 
 from twisted.application import internet, service
 from twisted.python import usage, log
@@ -49,7 +50,11 @@ def run():
         print '%s: %s' % (sys.argv[0], errortext)
         print '%s: Try --help for usage details.' % (sys.argv[0])
         sys.exit(1)
-    log.startLogging(sys.stdout)
+
+    # Start Logging
+    obs = StdOutObserver()
+    obs.start()
+
     s = makeService(options)
     s.startService()
     reactor.run()

swftp/logging.py

@@ -1,19 +1,46 @@
 """
 See COPYING for license information.
 """
+import sys
 import syslog as pysyslog
+
 from twisted.python import syslog
+from twisted.python import log
+
+WHITELISTED_LOG_SYSTEMS = ['SwFTP', '-']
+
+
+def msg(message, *args, **kwargs):
+    if not kwargs.get('system'):
+        kwargs['system'] = 'SwFTP'
+    return log.msg(message, *args, **kwargs)
+
+
+class LogObserver(object):
+    def start(self):
+        log.addObserver(self)
+
+    def stop(self):
+        log.removeObserver(self)
 
+    def __call__(self, event_dict):
+        if any((True for system in WHITELISTED_LOG_SYSTEMS
+                if event_dict.get('system', '').startswith(system))) \
+                or event_dict.get('isError', False):
+            self.obs.emit(event_dict)
+
+
+class StdOutObserver(LogObserver):
+    def __init__(self):
+        self.obs = log.FileLogObserver(sys.stdout)
 
-class SysLogObserver(object):
+
+class SysLogObserver(LogObserver):
     facility = pysyslog.LOG_USER
 
     def __init__(self):
         self.obs = syslog.SyslogObserver('swftp', facility=self.facility)
 
-    def __call__(self, event_dict):
-        self.obs.emit(event_dict)
-
 
 class LOG_USER(SysLogObserver):
     facility = pysyslog.LOG_USER

swftp/sftp/server.py

@@ -4,30 +4,35 @@
 See COPYING for license information.
 """
 from zope import interface
-import struct
 from collections import defaultdict
 
 from twisted.conch.interfaces import ISFTPServer, ISession
 from twisted.python import components, log
-from twisted.internet import defer
-
+from twisted.internet import defer, protocol
 from twisted.conch import avatar
 from twisted.conch.ssh import session
 from twisted.conch.ssh.filetransfer import (
     FileTransferServer, SFTPError, FX_FAILURE, FX_NO_SUCH_FILE)
 from twisted.conch.ssh.common import getNS
 from twisted.conch.ssh.transport import (
     SSHServerTransport, DISCONNECT_TOO_MANY_CONNECTIONS)
-from twisted.conch.ssh.connection import (
-    SSHConnection, MSG_CHANNEL_WINDOW_ADJUST)
 from twisted.conch.ssh.userauth import SSHUserAuthServer
+from twisted.conch.ssh.factory import SSHFactory
 
 from swftp.swift import NotFound, Conflict
+from swftp.logging import msg
 from swftp.sftp.swiftfile import SwiftFile
 from swftp.sftp.swiftdirectory import SwiftDirectory
 from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path
 
 
+class SwiftSSHFactory(SSHFactory):
+    def buildProtocol(self, addr):
+        t = protocol.Factory.buildProtocol(self, addr)
+        t.supportedPublicKeys = self.privateKeys.keys()
+        return t
+
+
 class SwiftSession(object):
     """ Barebones Session that closes when a client tries to open a shell.
         Provides t.c.i.ISession
@@ -51,25 +56,12 @@ def closed(self):
         pass
 
 
-class SwiftSSHConnection(SSHConnection):
-    transport = None
-
-    # SSHConnection is overridden to reduce verbosity.
-    def adjustWindow(self, channel, bytesToAdd):
-        if channel.localClosed:
-            return  # we're already closed
-        self.transport.sendPacket(MSG_CHANNEL_WINDOW_ADJUST, struct.pack('>2L',
-                                  self.channelsToRemoteChannel[channel],
-                                  bytesToAdd))
-        channel.localWindowLeft += bytesToAdd
-
-
 class SwiftFileTransferServer(FileTransferServer):
     client = None
     transport = None
 
     # Overridden to expose the session to the file object to do intellegent
-    # throttling. Without this, memory bloat occurs.
+    # throttling. Without this memory bloat occurs.
     def _cbOpenFile(self, fileObj, requestId):
         fileObj.session = self.transport.session
         FileTransferServer._cbOpenFile(self, fileObj, requestId)
@@ -108,34 +100,38 @@ def connectionLost(self, reason):
         log.msg(metric='num_clients', count=-1)
         if getattr(self, 'avatar', None):
             username = self.avatar.username()
-            log.msg("User Disconnected %s [%s/%s]" % (
-                    username,
-                    self._connCountMap[username],
-                    self.maxConnectionsPerUser,
-                    ))
+            msg("User Disconnected (%s) [%s/%s]" % (
+                username,
+                self._connCountMap[username],
+                self.maxConnectionsPerUser,
+                ))
             self._connCountMap[username] -= 1
             # To avoid a slow memory leak
             if self._connCountMap[username] == 0:
                 del self._connCountMap[username]
-        return super(SwiftSSHServerTransport, self).connectionLost(reason)
+
+        if self.service:
+            self.service.serviceStopped()
+        if hasattr(self, 'avatar'):
+            self.logoutFunction()
 
     def on_auth(self, res):
         if not getattr(self, 'avatar', None):
             return res
         username = self.avatar.username()
         self._connCountMap[username] += 1
-        log.msg("User Connected %s [%s/%s]" % (
-                username,
-                self._connCountMap[username],
-                self.maxConnectionsPerUser,
-                ))
+        msg("User Connected (%s) [%s/%s]" % (
+            username,
+            self._connCountMap[username],
+            self.maxConnectionsPerUser,
+            ))
         if self.maxConnectionsPerUser != 0 and \
                 self._connCountMap[username] > self.maxConnectionsPerUser:
-            log.msg("Too Many Connections For User %s [%s/%s]" % (
+            msg("Too Many Connections For User (%s) [%s/%s]" % (
                 username,
                 self._connCountMap[username],
                 self.maxConnectionsPerUser,
-            ))
+                ))
             self.sendDisconnect(
                 DISCONNECT_TOO_MANY_CONNECTIONS,
                 'too many connections')
@@ -186,9 +182,9 @@ def log_command(self, command, *args):
 
         """
         arg_list = ', '.join(str(arg) for arg in args)
-        log.msg("COMMAND: %s(%s)" % (command, arg_list),
-                system="SwFTP-SFTP, (%s)" % self.swiftconn.username,
-                metric='command.%s' % command)
+        msg("cmd.%s(%s)" % (command, arg_list),
+            system="SwFTP-SFTP, (%s)" % self.swiftconn.username,
+            metric='command.%s' % command)
 
 
 class SFTPServerForSwiftConchUser(object):
@@ -206,8 +202,6 @@ def __init__(self, avatar):
         self.conn = avatar.conn
         self.log_command('login')
 
-        # CHECK IF USER HAS TOO MANY CONNECTIONS
-
     def log_command(self, *args, **kwargs):
         """ Logs the given command.
 

swftp/sftp/service.py

@@ -4,6 +4,7 @@
 See COPYING for license information.
 """
 from swftp import VERSION
+from swftp.logging import StdOutObserver
 
 from twisted.application import internet, service
 from twisted.python import usage, log
@@ -51,7 +52,11 @@ def run():
         print '%s: %s' % (sys.argv[0], errortext)
         print '%s: Try --help for usage details.' % (sys.argv[0])
         sys.exit(1)
-    log.startLogging(sys.stdout)
+
+    # Start Logging
+    obs = StdOutObserver()
+    obs.start()
+
     s = makeService(options)
     s.startService()
     reactor.run()
@@ -98,13 +103,13 @@ def makeService(options):
     Makes a new swftp-sftp service. The only option is the config file
     location. See CONFIG_DEFAULTS for list of configuration options.
     """
-    from twisted.conch.ssh.factory import SSHFactory
     from twisted.conch.ssh.keys import Key
+    from twisted.conch.ssh.connection import SSHConnection
     from twisted.cred.portal import Portal
 
     from swftp.realm import SwftpRealm
     from swftp.sftp.server import (
-        SwiftSSHServerTransport, SwiftSSHConnection, SwiftSSHUserAuthServer)
+        SwiftSSHServerTransport, SwiftSSHUserAuthServer, SwiftSSHFactory)
     from swftp.auth import SwiftBasedAuthDB
     from swftp.utils import (
         log_runtime_info, GLOBAL_METRICS, parse_key_value_config)
@@ -167,14 +172,14 @@ def makeService(options):
     sftpportal = Portal(realm)
     sftpportal.registerChecker(authdb)
 
-    sshfactory = SSHFactory()
+    sshfactory = SwiftSSHFactory()
     protocol = SwiftSSHServerTransport
     protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user')
     sshfactory.protocol = protocol
     sshfactory.noisy = False
     sshfactory.portal = sftpportal
     sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer
-    sshfactory.services['ssh-connection'] = SwiftSSHConnection
+    sshfactory.services['ssh-connection'] = SSHConnection
 
     pub_key_string = file(c.get('sftp', 'pub_key')).read()
     priv_key_string = file(c.get('sftp', 'priv_key')).read()

swftp/sftp/swiftdirectory.py

@@ -18,7 +18,6 @@ def __init__(self, swiftfilesystem, fullpath):
                 ('.', {}),
                 ('..', {}),
             ])
-        self.done = False
 
     def get_full_listing(self):
         "Populate the directory listing."

swftp/utils.py

@@ -1,8 +1,8 @@
 """
 See COPYING for license information.
 """
-import time
 from collections import defaultdict
+import time
 
 from twisted.python import log
 from twisted.internet import reactor, tcp
@@ -75,9 +75,8 @@ class MetricCollector(object):
                             rolling aggregates.
 
     Example:
-        >>> import twisted.python.log
         >>> h = MetricCollector()
-        >>> twisted.python.log.addObserver(h.emit)
+        >>> h.start()
         >>> h.totals
         {}
         >>> log.msg(metric='my_metric')
@@ -89,6 +88,7 @@ class MetricCollector(object):
         >>> h.sample()
         >>> h.samples
         {'my_metric1': [1, 0]}
+        >>> h.stop()
 
     """
     def __init__(self, sample_size=10):