BOT: collector ftp(s)

intelmq/bots/collectors/ftp/collector_ftp.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""
+FTP collector bot
+
+Parameters:
+ftp_host: string
+ftp_port: number
+ftp_username, ftp_password: string
+ftp_directory: string
+ftp_file: string
+
+"""
+from __future__ import unicode_literals
+import sys
+from ftplib import FTP
+import socket
+import zipfile
+import io
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.harmonization import DateTime
+from intelmq.lib.message import Report
+
+
+class FTPCollectorBot(Bot):
+    def process(self):
+        self.logger.info("Downloading report from %s" %
+                         self.parameters.ftp_host + ':' +
+                         str(self.parameters.ftp_port))
+
+        ftp = FTP()
+        ftp.connect(host=self.parameters.ftp_host,
+                     port=self.parameters.ftp_port)
+        if hasattr(self.parameters, 'ftp_username') \
+                and hasattr(self.parameters, 'ftp_password'):
+            ftp.login(user=self.parameters.ftp_username,
+                       passwd=self.parameters.ftp_password)
+        cwd = '/'
+        if hasattr(self.parameters, 'ftp_directory'):
+            cwd = self.parameters.ftp_directory
+        ftp.cwd(cwd)
+        mem = io.BytesIO()
+        if hasattr(self.parameters, 'ftp_file'):
+            ftp.retrbinary("RETR " + self.parameters.ftp_file, mem.write)
+        else:
+            files = ftp.nlst(cwd)
+            if files:
+                #retrieve the last file in the directory
+                self.logger.info('Retrieving file: ' + files[-1])
+                ftp.retrbinary("RETR " + files[-1], mem.write)
+            else:
+                self.logger.error("No file found, terminating download")
+                return
+
+        self.logger.info("Report downloaded.")
+
+        raw_reports = []
+        try:
+            zfp = zipfile.ZipFile(mem, "r")
+        except zipfile.BadZipfile:
+            raw_reports.append(mem.getvalue())
+        else:
+            self.logger.info('Downloaded zip file, extracting following files: '
+                             + ', '.join(zfp.namelist()))
+            for filename in zfp.namelist():
+                raw_reports.append(zfp.read(filename))
+
+        for raw_report in raw_reports:
+            report = Report()
+            report.add("raw", raw_report, sanitize=True)
+            report.add("feed.name", self.parameters.feed, sanitize=True)
+            report.add("feed.url", 'ftp://' + self.parameters.ftp_host + ':' +
+                       str(self.parameters.ftp_port), sanitize=True)
+            report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
+            self.send_message(report)
+
+if __name__ == "__main__":
+    bot = FTPCollectorBot(sys.argv[1])
+    bot.start()

intelmq/bots/collectors/ftp/collector_ftps.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+FTPS collector bot
+
+Parameters:
+ftps_host: string
+ftps_port: number
+ftps_username, ftps_password: string
+ftps_directory: string
+ftps_file: string
+
+
+"""
+from __future__ import unicode_literals
+import sys
+from ftplib import FTP_TLS
+import socket
+import ssl
+import zipfile
+import io
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.harmonization import DateTime
+from intelmq.lib.message import Report
+
+
+# BEGIN content from Stack Overflow
+# Original question: https://stackoverflow.com/questions/12164470/python-ftp-implicit-tls-connection-issue
+# Question author (Martin Prikryl): https://stackoverflow.com/users/850848/martin-prikryl
+# Answer author (Grzegorz Wierzowiecki): https://stackoverflow.com/users/544721/grzegorz-wierzowiecki
+class FTPS(FTP_TLS):
+    def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
+                 certfile=None, timeout=60):
+        FTP_TLS.__init__(self, host, user, passwd, acct, keyfile, certfile,
+                         timeout)
+
+    def connect(self, host='', port=0, timeout=-999):
+        if host != '':
+            self.host = host
+        if port > 0:
+            self.port = port
+        if timeout != -999:
+            self.timeout = timeout
+
+        self.sock = socket.create_connection((self.host, self.port),
+                                             self.timeout)
+        self.af = self.sock.family
+        self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
+                                    ssl_version=ssl.PROTOCOL_TLSv1)
+        self.file = self.sock.makefile('rb')
+        self.welcome = self.getresp()
+
+        return self.welcome
+# END content from Stack Overflow
+
+class FTPSCollectorBot(Bot):
+    def process(self):
+        self.logger.info("Downloading report from %s" %
+                         self.parameters.ftps_host + ':' +
+                         str(self.parameters.ftps_port))
+
+        ftps = FTPS()
+        ftps.connect(host=self.parameters.ftps_host,
+                     port=self.parameters.ftps_port)
+        if hasattr(self.parameters, 'ftps_username') \
+                and hasattr(self.parameters, 'ftps_password'):
+            ftps.login(user=self.parameters.ftps_username,
+                       passwd=self.parameters.ftps_password)
+        ftps.prot_p()
+        if hasattr(self.parameters, 'ftps_directory'):
+            ftps.cwd(self.parameters.ftps_directory)
+        mem = io.BytesIO()
+        ftps.retrbinary("RETR " + self.parameters.ftps_file, mem.write)
+
+        self.logger.info("Report downloaded.")
+
+        raw_reports = []
+        try:
+            zfp = zipfile.ZipFile(mem, "r")
+        except zipfile.BadZipfile:
+            raw_reports.append(mem.getvalue())
+        else:
+            self.logger.info('Downloaded zip file, extracting following files: '
+                             + ', '.join(zfp.namelist()))
+            for filename in zfp.namelist():
+                raw_reports.append(zfp.read(filename))
+
+        for raw_report in raw_reports:
+            report = Report()
+            report.add("raw", raw_report, sanitize=True)
+            report.add("feed.name", self.parameters.feed, sanitize=True)
+            report.add("feed.url", 'ftps://' + self.parameters.ftps_host + ':' +
+                       str(self.parameters.ftps_port), sanitize=True)
+            report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
+            self.send_message(report)
+
+if __name__ == "__main__":
+    bot = FTPSCollectorBot(sys.argv[1])
+    bot.start()

intelmq/tests/bots/collectors/ftp/test_collector_ftp.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Testing FTP collector
+"""
+from __future__ import unicode_literals
+
+import intelmq.bots.collectors.ftp.collector_ftp

intelmq/tests/bots/collectors/ftp/test_collector_ftps.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Testing FTPS collector
+"""
+from __future__ import unicode_literals
+
+import intelmq.bots.collectors.ftp.collector_ftps