From cebf2d3c2b4bb8fd5e699adb24e5bb30584bd001 Mon Sep 17 00:00:00 2001 From: Kevin McDonald Date: Fri, 8 Nov 2013 14:55:52 -0600 Subject: [PATCH] Adds filtering to log observers At the moment, it only lets through log events with a system that starts with 'SwFTP' and '-' * swftp.logging.StdOutObserver - outputs logging to stdout * swftp.logging.LOG_USER, swftp.logging.LOG_LOCAL0, etc. - Outputs to log facility --- README.md | 4 +-- swftp/ftp/server.py | 22 ++++++++---- swftp/ftp/service.py | 7 +++- swftp/logging.py | 35 ++++++++++++++++--- swftp/sftp/server.py | 68 ++++++++++++++++-------------------- swftp/sftp/service.py | 15 +++++--- swftp/sftp/swiftdirectory.py | 1 - swftp/utils.py | 6 ++-- 8 files changed, 99 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 6f671b5..bc8e602 100644 --- a/README.md +++ b/README.md @@ -209,8 +209,8 @@ Packaged with SwFTP are a set of example init scripts, upstart scripts. They are * /etc/init/swftp-ftp.conf * /etc/init/swftp-sftp.conf * init.d - * /etc/init/swftp-ftp - * /etc/init/swftp-sftp + * /etc/init.d/swftp-ftp + * /etc/init.d/swftp-sftp * Supervisor * /etc/supervisor/conf.d/swftp.conf * Example swftp.conf file diff --git a/swftp/ftp/server.py b/swftp/ftp/server.py index 7281ea2..2ea5d29 100644 --- a/swftp/ftp/server.py +++ b/swftp/ftp/server.py @@ -15,6 +15,7 @@ from twisted.internet.protocol import Protocol from twisted.python import log +from swftp.logging import msg from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path from swftp.swift import NotFound, Conflict @@ -56,6 +57,11 @@ def connectionLost(self, *args, **kwargs): if self.shell: username = self.shell.username() + msg("User Disconnected (%s) [%s/%s]" % ( + username, + self._connCountMap[username], + self.maxConnectionsPerUser, + )) self._connCountMap[username] -= 1 # To avoid a slow memory leak if self._connCountMap[username] == 0: @@ -69,13 +75,18 @@ def ftp_PASS(self, *args, **kwargs): def pass_cb(res): username = self.shell.username() self._connCountMap[username] += 1 + msg("User Connected (%s) [%s/%s]" % ( + username, + self._connCountMap[username], + self.maxConnectionsPerUser, + )) if self.maxConnectionsPerUser != 0 and \ self._connCountMap[username] > self.maxConnectionsPerUser: - log.msg("Too Many Connections For User %s [%s/%s]" % ( + msg("Too Many Connections For User (%s) [%s/%s]" % ( username, self._connCountMap[username], self.maxConnectionsPerUser, - )) + )) self.sendLine(RESPONSE[TOO_MANY_CONNECTIONS]) self.transport.loseConnection() return res @@ -124,9 +135,9 @@ def __init__(self, swiftconn): def log_command(self, command, *args): arg_list = ', '.join(str(arg) for arg in args) - log.msg("COMMAND: %s(%s)" % (command, arg_list), - system="SwFTP-FTP, (%s)" % self.swiftconn.username, - metric='command.%s' % command) + msg("cmd: %s(%s)" % (command, arg_list), + system="SwFTP-FTP, (%s)" % self.swiftconn.username, + metric='command.%s' % command) def username(self): return self.swiftconn.username @@ -274,7 +285,6 @@ def openForWriting(self, path): fullpath = self._fullpath(path) container, obj = obj_to_path(fullpath) if not container or not obj: - log.msg('cannot upload to root') raise CmdNotImplementedForArgError( 'Cannot upload files to root directory.') f = SwiftWriteFile(self.swiftfilesystem, fullpath) diff --git a/swftp/ftp/service.py b/swftp/ftp/service.py index 5a2034c..75dee1d 100644 --- a/swftp/ftp/service.py +++ b/swftp/ftp/service.py @@ -4,6 +4,7 @@ See COPYING for license information. """ from swftp import VERSION +from swftp.logging import StdOutObserver from twisted.application import internet, service from twisted.python import usage, log @@ -49,7 +50,11 @@ def run(): print '%s: %s' % (sys.argv[0], errortext) print '%s: Try --help for usage details.' % (sys.argv[0]) sys.exit(1) - log.startLogging(sys.stdout) + + # Start Logging + obs = StdOutObserver() + obs.start() + s = makeService(options) s.startService() reactor.run() diff --git a/swftp/logging.py b/swftp/logging.py index 0b5f403..6de69ca 100644 --- a/swftp/logging.py +++ b/swftp/logging.py @@ -1,19 +1,46 @@ """ See COPYING for license information. """ +import sys import syslog as pysyslog + from twisted.python import syslog +from twisted.python import log + +WHITELISTED_LOG_SYSTEMS = ['SwFTP', '-'] + + +def msg(message, *args, **kwargs): + if not kwargs.get('system'): + kwargs['system'] = 'SwFTP' + return log.msg(message, *args, **kwargs) + + +class LogObserver(object): + def start(self): + log.addObserver(self) + + def stop(self): + log.removeObserver(self) + def __call__(self, event_dict): + if any((True for system in WHITELISTED_LOG_SYSTEMS + if event_dict.get('system', '').startswith(system))) \ + or event_dict.get('isError', False): + self.obs.emit(event_dict) + + +class StdOutObserver(LogObserver): + def __init__(self): + self.obs = log.FileLogObserver(sys.stdout) -class SysLogObserver(object): + +class SysLogObserver(LogObserver): facility = pysyslog.LOG_USER def __init__(self): self.obs = syslog.SyslogObserver('swftp', facility=self.facility) - def __call__(self, event_dict): - self.obs.emit(event_dict) - class LOG_USER(SysLogObserver): facility = pysyslog.LOG_USER diff --git a/swftp/sftp/server.py b/swftp/sftp/server.py index 3da1190..3b15565 100644 --- a/swftp/sftp/server.py +++ b/swftp/sftp/server.py @@ -4,13 +4,11 @@ See COPYING for license information. """ from zope import interface -import struct from collections import defaultdict from twisted.conch.interfaces import ISFTPServer, ISession from twisted.python import components, log -from twisted.internet import defer - +from twisted.internet import defer, protocol from twisted.conch import avatar from twisted.conch.ssh import session from twisted.conch.ssh.filetransfer import ( @@ -18,16 +16,23 @@ from twisted.conch.ssh.common import getNS from twisted.conch.ssh.transport import ( SSHServerTransport, DISCONNECT_TOO_MANY_CONNECTIONS) -from twisted.conch.ssh.connection import ( - SSHConnection, MSG_CHANNEL_WINDOW_ADJUST) from twisted.conch.ssh.userauth import SSHUserAuthServer +from twisted.conch.ssh.factory import SSHFactory from swftp.swift import NotFound, Conflict +from swftp.logging import msg from swftp.sftp.swiftfile import SwiftFile from swftp.sftp.swiftdirectory import SwiftDirectory from swftp.swiftfilesystem import SwiftFileSystem, swift_stat, obj_to_path +class SwiftSSHFactory(SSHFactory): + def buildProtocol(self, addr): + t = protocol.Factory.buildProtocol(self, addr) + t.supportedPublicKeys = self.privateKeys.keys() + return t + + class SwiftSession(object): """ Barebones Session that closes when a client tries to open a shell. Provides t.c.i.ISession @@ -51,25 +56,12 @@ def closed(self): pass -class SwiftSSHConnection(SSHConnection): - transport = None - - # SSHConnection is overridden to reduce verbosity. - def adjustWindow(self, channel, bytesToAdd): - if channel.localClosed: - return # we're already closed - self.transport.sendPacket(MSG_CHANNEL_WINDOW_ADJUST, struct.pack('>2L', - self.channelsToRemoteChannel[channel], - bytesToAdd)) - channel.localWindowLeft += bytesToAdd - - class SwiftFileTransferServer(FileTransferServer): client = None transport = None # Overridden to expose the session to the file object to do intellegent - # throttling. Without this, memory bloat occurs. + # throttling. Without this memory bloat occurs. def _cbOpenFile(self, fileObj, requestId): fileObj.session = self.transport.session FileTransferServer._cbOpenFile(self, fileObj, requestId) @@ -108,34 +100,38 @@ def connectionLost(self, reason): log.msg(metric='num_clients', count=-1) if getattr(self, 'avatar', None): username = self.avatar.username() - log.msg("User Disconnected %s [%s/%s]" % ( - username, - self._connCountMap[username], - self.maxConnectionsPerUser, - )) + msg("User Disconnected (%s) [%s/%s]" % ( + username, + self._connCountMap[username], + self.maxConnectionsPerUser, + )) self._connCountMap[username] -= 1 # To avoid a slow memory leak if self._connCountMap[username] == 0: del self._connCountMap[username] - return super(SwiftSSHServerTransport, self).connectionLost(reason) + + if self.service: + self.service.serviceStopped() + if hasattr(self, 'avatar'): + self.logoutFunction() def on_auth(self, res): if not getattr(self, 'avatar', None): return res username = self.avatar.username() self._connCountMap[username] += 1 - log.msg("User Connected %s [%s/%s]" % ( - username, - self._connCountMap[username], - self.maxConnectionsPerUser, - )) + msg("User Connected (%s) [%s/%s]" % ( + username, + self._connCountMap[username], + self.maxConnectionsPerUser, + )) if self.maxConnectionsPerUser != 0 and \ self._connCountMap[username] > self.maxConnectionsPerUser: - log.msg("Too Many Connections For User %s [%s/%s]" % ( + msg("Too Many Connections For User (%s) [%s/%s]" % ( username, self._connCountMap[username], self.maxConnectionsPerUser, - )) + )) self.sendDisconnect( DISCONNECT_TOO_MANY_CONNECTIONS, 'too many connections') @@ -186,9 +182,9 @@ def log_command(self, command, *args): """ arg_list = ', '.join(str(arg) for arg in args) - log.msg("COMMAND: %s(%s)" % (command, arg_list), - system="SwFTP-SFTP, (%s)" % self.swiftconn.username, - metric='command.%s' % command) + msg("cmd.%s(%s)" % (command, arg_list), + system="SwFTP-SFTP, (%s)" % self.swiftconn.username, + metric='command.%s' % command) class SFTPServerForSwiftConchUser(object): @@ -206,8 +202,6 @@ def __init__(self, avatar): self.conn = avatar.conn self.log_command('login') - # CHECK IF USER HAS TOO MANY CONNECTIONS - def log_command(self, *args, **kwargs): """ Logs the given command. diff --git a/swftp/sftp/service.py b/swftp/sftp/service.py index e2c6795..eed5f95 100644 --- a/swftp/sftp/service.py +++ b/swftp/sftp/service.py @@ -4,6 +4,7 @@ See COPYING for license information. """ from swftp import VERSION +from swftp.logging import StdOutObserver from twisted.application import internet, service from twisted.python import usage, log @@ -51,7 +52,11 @@ def run(): print '%s: %s' % (sys.argv[0], errortext) print '%s: Try --help for usage details.' % (sys.argv[0]) sys.exit(1) - log.startLogging(sys.stdout) + + # Start Logging + obs = StdOutObserver() + obs.start() + s = makeService(options) s.startService() reactor.run() @@ -98,13 +103,13 @@ def makeService(options): Makes a new swftp-sftp service. The only option is the config file location. See CONFIG_DEFAULTS for list of configuration options. """ - from twisted.conch.ssh.factory import SSHFactory from twisted.conch.ssh.keys import Key + from twisted.conch.ssh.connection import SSHConnection from twisted.cred.portal import Portal from swftp.realm import SwftpRealm from swftp.sftp.server import ( - SwiftSSHServerTransport, SwiftSSHConnection, SwiftSSHUserAuthServer) + SwiftSSHServerTransport, SwiftSSHUserAuthServer, SwiftSSHFactory) from swftp.auth import SwiftBasedAuthDB from swftp.utils import ( log_runtime_info, GLOBAL_METRICS, parse_key_value_config) @@ -167,14 +172,14 @@ def makeService(options): sftpportal = Portal(realm) sftpportal.registerChecker(authdb) - sshfactory = SSHFactory() + sshfactory = SwiftSSHFactory() protocol = SwiftSSHServerTransport protocol.maxConnectionsPerUser = c.getint('sftp', 'sessions_per_user') sshfactory.protocol = protocol sshfactory.noisy = False sshfactory.portal = sftpportal sshfactory.services['ssh-userauth'] = SwiftSSHUserAuthServer - sshfactory.services['ssh-connection'] = SwiftSSHConnection + sshfactory.services['ssh-connection'] = SSHConnection pub_key_string = file(c.get('sftp', 'pub_key')).read() priv_key_string = file(c.get('sftp', 'priv_key')).read() diff --git a/swftp/sftp/swiftdirectory.py b/swftp/sftp/swiftdirectory.py index 5d8fa84..b2aa3a3 100644 --- a/swftp/sftp/swiftdirectory.py +++ b/swftp/sftp/swiftdirectory.py @@ -18,7 +18,6 @@ def __init__(self, swiftfilesystem, fullpath): ('.', {}), ('..', {}), ]) - self.done = False def get_full_listing(self): "Populate the directory listing." diff --git a/swftp/utils.py b/swftp/utils.py index d942f87..ad454ec 100644 --- a/swftp/utils.py +++ b/swftp/utils.py @@ -1,8 +1,8 @@ """ See COPYING for license information. """ -import time from collections import defaultdict +import time from twisted.python import log from twisted.internet import reactor, tcp @@ -75,9 +75,8 @@ class MetricCollector(object): rolling aggregates. Example: - >>> import twisted.python.log >>> h = MetricCollector() - >>> twisted.python.log.addObserver(h.emit) + >>> h.start() >>> h.totals {} >>> log.msg(metric='my_metric') @@ -89,6 +88,7 @@ class MetricCollector(object): >>> h.sample() >>> h.samples {'my_metric1': [1, 0]} + >>> h.stop() """ def __init__(self, sample_size=10):