diff --git a/README.md b/README.md index cb4f5a6..15cea2b 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,12 @@ This is a simple bot which fetches RSS feeds and posts them to an IRC channel. - Fetches every feed in a separate thread - Posts new news items to an IRC channel - Sends information via private messages +- Full utf-8 support +- Nick login support +- Post delayed during conversation +- Keywords in news title filtering support +- Customizable post colors +- Automatic join to channel on kick # Bot's commands: diff --git a/bot.py b/bot.py index 8455a7c..472d120 100644 --- a/bot.py +++ b/bot.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- import ssl @@ -9,6 +9,7 @@ import tinyurl import time import re +import sys import feedparser import datetime import dateutil.parser @@ -17,6 +18,7 @@ from config import Config from feedupdater import FeedUpdater + class IRCBot(irc.bot.SingleServerIRCBot): def __init__(self, config, db, on_connect_cb): self.__config = config @@ -25,44 +27,59 @@ def __init__(self, config, db, on_connect_cb): self.__servers = [irc.bot.ServerSpec(self.__config.HOST, self.__config.PORT, self.__config.PASSWORD)] self.__first_start = False self.color_num = self.__config.num_col - self.color_date = self.__config.date self.color_feedname = self.__config.feedname + self.color_newstitle = self.__config.newstitle self.color_url = self.__config.url + self.color_date = self.__config.date self.shorturls = self.__config.shorturls self.dateformat = self.__config.dateformat + self.filterkeywords = self.__config.filterkeywords if self.__config.SSL: ssl_factory = irc.connection.Factory(wrapper=ssl.wrap_socket) + print datetime.datetime.now(), u"Starting SSL connection." + sys.stdout.flush() super(IRCBot, self).__init__(self.__servers, self.__config.NICK, self.__config.NICK, connect_factory=ssl_factory) + else: + print datetime.datetime.now(), u"Starting connection." + sys.stdout.flush() super(IRCBot, self).__init__(self.__servers, self.__config.NICK, self.__config.NICK) - def on_kick(self, connection, event): - """Join the correct channel again""" - if irc.client.is_channel(self.__config.CHANNEL): - time.sleep(31) - connection.join(self.__config.CHANNEL) - def on_welcome(self, connection, event): - """Authenticate using NickServ""" + """Login""" if self.__config.NICKPASS: - self.connection.privmsg("NickServ", "IDENTIFY {}".format(self.__config.NICKPASS)) + print datetime.datetime.now(), u"Starting authentication." + sys.stdout.flush() + self.send_msg("NickServ", "IDENTIFY {}".format(self.__config.NICKPASS)) + """Join the correct channel upon connecting""" if irc.client.is_channel(self.__config.CHANNEL): + print datetime.datetime.now(), u"Joining to channel." + sys.stdout.flush() connection.join(self.__config.CHANNEL) def on_join(self, connection, event): """Say hello to other people in the channel. """ - welcome_msg = "Hi, I'm " + self.__get_colored_text('3',str(connection.get_nickname())) + " your bot. Send " + self.__get_colored_text(self.color_num,"!help") +" to get a list of commands." - if not self.__first_start: - connection.privmsg(self.__config.CHANNEL, welcome_msg) + self.send_msg(self.__config.CHANNEL, self.welcome_msg()) self.__on_connect_cb() self.__first_start = True - if event.source.nick != connection.get_nickname(): - connection.privmsg(event.source.nick, welcome_msg) + def welcome_msg(self): + msg = u"Hi, I'm the channel's " + self.get_bolded_text(self.__get_colored_text(self.color_feedname,"RSS")) + u" news publishing bot v2.0. Send " + self.__get_colored_text(self.color_num,"!help") + u" to receive a list of commands in private message (PM). If you find me annoying, you can to use " + self.__get_colored_text(self.color_num,"/IGNORE " + self.connection.get_nickname()) + u" to stop reading me." + time.sleep(1) + return msg + def on_kick(self, connection, event): + """Join the correct channel again""" + banned_nick = event.arguments[0].lower().strip() + botnick = self.connection.get_nickname().lower() + if irc.client.is_channel(self.__config.CHANNEL) and banned_nick == botnick: + time.sleep(31) + print datetime.datetime.now(), u"Joining to channel again." + sys.stdout.flush() + connection.join(self.__config.CHANNEL) def __handle_msg(self, msg): """Handles a cmd private message.""" @@ -75,13 +92,13 @@ def __handle_msg(self, msg): elif msg == "!list": answer = "" for entry in self.__db.get_feeds(): - answer += "#" + self.__get_colored_text(self.color_num,str(entry[0])) + ": " + entry[1] + ", " + self.__get_colored_text(self.color_url,str(entry[2])) + self.__get_colored_text(self.color_date,", updated every ") + self.__get_colored_text(self.color_num,str(entry[3])) + self.__get_colored_text(self.color_date," min") + "\n" + answer += "#" + self.__get_colored_text(self.color_num,str(entry[0]) + " - ") + self.get_bolded_text(self.__get_colored_text(self.color_feedname,entry[1] + " > ")) + self.__get_colored_text(self.color_url,entry[2] + ",") + u" updated every " + self.__get_colored_text(self.color_num,str(entry[3])) + u" minutes." + "\n" # Print some simple stats (Feed / News count) elif msg == "!stats": feeds_count = self.__db.get_feeds_count() news_count = self.__db.get_news_count() - answer = "Feeds: " + self.__get_colored_text(self.color_num,str(feeds_count)) + ", News: " + self.__get_colored_text(self.color_num,str(news_count)) + answer = u"Feeds: " + self.get_bolded_text(self.__get_colored_text(self.color_num,str(feeds_count))) + u", News: " + self.get_bolded_text(self.__get_colored_text(self.color_num,str(news_count))) # Print last config.feedlimit news. elif msg == "!last": @@ -91,7 +108,7 @@ def __handle_msg(self, msg): items = items[::-1] for entry in items: - answer += "#" + self.__get_colored_text(self.color_num,str(entry[0])) + ": " + entry[1] + ", " + self.__get_colored_text(self.color_url,str(entry[2])) + ", " + self.__get_colored_text(self.color_date,str(entry[3])) + "\n" + answer += "#" + self.__get_colored_text(self.color_num,str(entry[0]) + " - ") + self.get_bolded_text(self.__get_colored_text(self.color_newstitle,entry[1] + " > ")) + self.__get_colored_text(self.color_url,entry[2] + ", ") + self.__get_colored_text(self.color_date,str(entry[3])) + "\n" # Print last config.feedlimit news for a specific feed elif msg.startswith("!lastfeed"): @@ -99,20 +116,20 @@ def __handle_msg(self, msg): try: feedid = int(msg.replace("!lastfeed","").strip()) except: - return self.__get_colored_text('1',"Wrong command: ") + msg + ", use: !lastfeed " + return self.__get_colored_text('1',u"Wrong command. ") + msg + u". Send !lastfeed " items = self.__db.get_news_from_feed(feedid, self.__config.feedlimit) if not self.__config.feedorderdesc: items = items[::-1] for entry in items: - answer += "#" + self.__get_colored_text(self.color_num,str(entry[0])) + ": " + entry[1] + ", " + self.__get_colored_text(self.color_url,str(entry[2])) + ", " + self.__get_colored_text(self.color_date,str(entry[3])) + "\n" + answer += "#" + self.__get_colored_text(self.color_num,str(entry[0]) + " - ") + self.get_bolded_text(self.__get_colored_text(self.color_newstitle,entry[1] + " > ")) + self.__get_colored_text(self.color_url,entry[2] + ", ") + self.__get_colored_text(self.color_date,str(entry[3])) + "\n" # Else tell the user how to use the bot else: - answer = "Use !help for possible commands." + answer = u"Send !help to see the available commands." except Exception as e: - print e - answer = "Something went wrong :(" - + print datetime.datetime.now(), e + sys.stdout.flush() + answer = u"Something was wrong." return answer def on_privmsg(self, connection, event): @@ -122,22 +139,26 @@ def on_privmsg(self, connection, event): # Get the message and return an answer msg = event.arguments[0].lower().strip() - print msg - + print datetime.datetime.now(), msg, "command from", event.source.nick + sys.stdout.flush() answer = self.__handle_msg(msg) self.send_msg(event.source.nick, answer) + time.sleep(5) def on_pubmsg(self, connection, event): + Config.lastpubmsg = time.time() """Handles the bot's public (channel) messages""" if len(event.arguments) < 1: return - - # Get the message. We are only interested in "!help" + # Get the message. We are only interested in "!help" or botnick msg = event.arguments[0].lower().strip() - + botnick = self.connection.get_nickname() # Send the answer as a private message if msg == "!help": self.send_msg(event.source.nick, self.__help_msg()) + # Send the answer as a public message + if botnick.lower() in msg: + self.send_msg(self.__config.CHANNEL, self.welcome_msg()) def on_nicknameinuse(self, connection, event): """Changes the nickname if necessary""" @@ -153,15 +174,35 @@ def send_msg(self, target, msg): self.connection.privmsg(target, sub_line) time.sleep(1) # Don't flood the target except Exception as e: - print e + print datetime.datetime.now(), e + sys.stdout.flush() def post_news(self, feed_name, title, url, date): + """Cancel post if filter keyword is in title""" + for keyword in self.filterkeywords: + if keyword in title.lower(): + print datetime.datetime.now(), u"Found", keyword, "keyword in title. Aborting post." + sys.stdout.flush() + return + """Try shortening url""" + if self.__config.shorturls: + try: + post_url = tinyurl.create_one(url) + if ("error" in post_url.lower()): + post_url = url + except Exception as e: + post_url = url + print datetime.datetime.now(), e + sys.stdout.flush() + else: + post_url = url """Posts a new announcement to the channel""" try: - msg = self.__get_colored_text(self.color_feedname,str(feed_name)) + ": " + title + ", " + self.__get_colored_text(self.color_url,url) + ", " + self.__get_colored_text(self.color_date,str(date)) + msg = self.__get_colored_text(self.color_feedname,feed_name + ": ") + self.get_bolded_text(self.__get_colored_text(self.color_newstitle,title)) + " > " + self.__get_colored_text(self.color_url,post_url + ", ") + self.__get_colored_text(self.color_date,str(date)) self.send_msg(self.__config.CHANNEL, msg) except Exception as e: - print e + print datetime.datetime.now(), e + sys.stdout.flush() def __get_colored_text(self, color, text): if not self.__config.use_colors: @@ -169,18 +210,25 @@ def __get_colored_text(self, color, text): return Colours(color, text).get() + def get_bolded_text(self, string): + """Returns the string bolded.""" + return "\002" + string + "\002" + def __help_msg(self): """Returns the help/usage message""" - return """\ + return u"""\ Help: - Send all commands as a private message to """ + self.connection.get_nickname() + """ - - !help Prints this help - - !list Prints all feeds - - !stats Prints some statistics - - !last Prints the last 10 entries - - !lastfeed Prints the last 10 entries from a specific feed + - /IGNORE """ + self.connection.get_nickname() + u""" - Lets you stop reading the bot. + + You can send these commands in private message (PM) to """ + self.connection.get_nickname() + u""": + - !help - Show this help message. + - !stats - Show some statistics. + - !list - Show all feeds. + - !last - Show last news published in all feeds. + - !lastfeed - Show last news published in a specific feed. """ + class Bot(object): def __init__(self): self.__config = Config() @@ -193,9 +241,9 @@ def __init__(self): self.__connected = False def __check_config(self): - necessary_options = ["HOST", "PORT", "PASSWORD", "SSL", "CHANNEL", "NICK", "admin_nicks", "use_colors", - "num_col", "date", "feedname", "shorturls", "dateformat", "feedlimit", "update_before_connecting", - "url", "feedorderdesc"] + necessary_options = ["HOST", "PORT", "PASSWORD", "SSL", "CHANNEL", "NICK", "admin_nicks", "use_colors", + "num_col", "date", "feedname", "shorturls", "dateformat", "feedlimit", "update_before_connecting", + "url", "feedorderdesc"] missing_options = [] for key in necessary_options: if not hasattr(self.__config, key): @@ -207,21 +255,24 @@ def get_missing_options(self): def start(self): """Starts the IRC bot""" + print datetime.datetime.now(), u"Starting bot." + sys.stdout.flush() threading.Thread(target=self.__irc.start).start() def initial_feed_update(self): def print_feed_update(feed_title, news_title, news_url, news_date): - print(u"[+]: {}||{}||{}||{}".format(feed_title, news_title, news_url, news_date)) + print datetime.datetime.now(), u"[+]: {}||{}||{}||{}".format(feed_title, news_title, news_url, news_date) + sys.stdout.flush() if self.__config.update_before_connecting: - print "Started pre-connection updates!" + print datetime.datetime.now(), u"Starting offline update." + sys.stdout.flush() self.__feedupdater.update_feeds(print_feed_update, False) - print "DONE!" def on_started(self): """Gets executed after the IRC thread has successfully established a connection.""" if not self.__connected: - print "Connected!" + print datetime.datetime.now(), u"Starting feeds periodic update..." + sys.stdout.flush() self.__feedupdater.update_feeds(self.__irc.post_news, True) - print "Started feed updates!" self.__connected = True diff --git a/colour.py b/colour.py index 3b1037e..931414e 100644 --- a/colour.py +++ b/colour.py @@ -1,37 +1,51 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- class Colours: def __init__(self, col, string): self.colour = col self.string = string - self.default = '\017' - self.ret = self.string+self.default + self.default = '\003' + self.ret = self.string + self.default def get(self): - if self.colour == '1' or self.colour == 'red': - return '\00304'+self.ret - elif self.colour == '2' or self.colour == 'green': - return '\00303'+self.ret - elif self.colour == '3' or self.colour == 'yellow': - return '\00308'+self.ret - elif self.colour == '4' or self.colour == 'blue': - return '\00312'+self.ret - elif self.colour == '5' or self.colour == 'purple': - return '\00306'+self.ret - elif self.colour == '6' or self.colour == 'cyan': - return '\00310'+self.ret - elif self.colour == '7' or self.colour == 'lightgreen': - return '\00309'+self.ret - elif self.colour == '8' or self.colour == 'grey': - return '\00314'+self.ret - elif self.colour == '9' or self.colour == 'pink': - return '\00313'+self.ret - elif self.colour == '10' or self.colour == 'lightblue': - return '\00311'+self.ret + if self.colour == '0' or self.colour == 'white': + return '\00300' + self.ret + elif self.colour == '1' or self.colour == 'black': + return '\00301' + self.ret + elif self.colour == '2' or self.colour == 'blue': + return '\00302' + self.ret + elif self.colour == '3' or self.colour == 'green': + return '\00303' + self.ret + elif self.colour == '4' or self.colour == 'red': + return '\00304' + self.ret + elif self.colour == '5' or self.colour == 'brown': + return '\00305' + self.ret + elif self.colour == '6' or self.colour == 'purple': + return '\00306' + self.ret + elif self.colour == '7' or self.colour == 'orange': + return '\00307' + self.ret + elif self.colour == '8' or self.colour == 'yellow': + return '\00308' + self.ret + elif self.colour == '9' or self.colour == 'lightgreen': + return '\00309' + self.ret + elif self.colour == '10' or self.colour == 'teal': + return '\00310' + self.ret + elif self.colour == '11' or self.colour == 'cyan': + return '\00311' + self.ret + elif self.colour == '12' or self.colour == 'lightblue': + return '\00312' + self.ret + elif self.colour == '13' or self.colour == 'pink': + return '\00313' + self.ret + elif self.colour == '14' or self.colour == 'grey': + return '\00314' + self.ret + elif self.colour == '15' or self.colour == 'lightgrey': + return '\00315' + self.ret else: - return '\00316'+self.ret + return self.string """ if __name__ == "__main__": for i in range(0, 11): - print Colours(str(i), 'Testing').get() + "TESTING "+Colours(str(i), 'wat').get() + print Colours(str(i), 'Testing').get() + "TESTING " + Colours(str(i), 'wat').get() """ diff --git a/config.py.sample b/config.py.sample index 28f13ca..57c3fc5 100644 --- a/config.py.sample +++ b/config.py.sample @@ -1,28 +1,37 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- class Config(object): + lastpubmsg = 0 + def __init__(self): - self.HOST = "irc.freenode.org" - self.PORT = 6667 + self.HOST = "irc.rizon.net" + self.PORT = 9999 self.PASSWORD = None - self.SSL = False - self.CHANNEL = "##YOURCHANNEL" - self.NICK = "YOURBOTNICK" - self.NICKPASS = "YOURPASSWORD" - self.admin_nicks= ['YOURADMINNICK'] - #=Colours= - #1 - red 2 - green 3 - yellow - #4 - blue 5 - purple 6 - cyan - #7 - lightgreen 8 - grey 9 - pink - #10 - lighblue + self.SSL = True + self.NICK = "rss2irc" + self.NICKPASS = "" + self.CHANNEL = "#tests" + self.admin_nicks= [''] + + """ + 00 - white 01 - black 02 - blue (navy) 03 - green + 04 - red 05 - brown 06 - purple 07 - orange + 08 - yellow 09 - light green 10 - teal 11 - cyan + 12 - light blue 13 - pink 14 - grey 15 - light grey + """ + self.use_colors = True - self.num_col = '1' - self.date = '8' - self.feedname = '2' - self.url = '4' - self.shorturls = False + self.num_col = 'red' # Empty for dafault color + self.feedname = 'orange' # Empty for dafault color + self.newstitle = '' # Empty for dafault color + self.url = 'teal' # Empty for dafault color + self.date = 'grey' # Empty for dafault color + self.shorturls = True self.dateformat = '%Y-%m-%d %H:%M:%S %z' self.feedlimit = 10 - self.feedorderdesc = True - self.update_before_connecting = True #Update all feeds before connecting to the IRC server + self.postdelay = 180 # Post delay during conversation in seconds. 0 for off. + self.feedorderdesc = False + self.update_before_connecting = True # Update all feeds before connecting to the IRC server + self.filterkeywords = {} # Lower-case keywords list to filter in title news. Use + # {"one","two","three"} to activate or {} for deactivate. diff --git a/db.py b/db.py index 2622e9f..b9c9575 100644 --- a/db.py +++ b/db.py @@ -1,8 +1,12 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- from sqlite3worker import Sqlite3Worker +from config import Config import os +import sys +import datetime +import time class FeedDB(object): def __init__(self, config): @@ -10,6 +14,7 @@ def __init__(self, config): self.__db_worker = None self.__config = config self.__initiate_db() + self.__postdelay = self.__config.postdelay def __initiate_db(self): """Create a DB connection""" @@ -59,9 +64,14 @@ def get_news_count(self): return count def insert_news(self, feed_id, title, url, published): - """Checks if a news item with the given information exists. If not, create a new entry.""" - exists = self.__db_worker.execute("select exists(select 1 FROM news WHERE feedid = :feedid and url = :url and published = :published LIMIT 1)", {'feedid': feed_id, 'url': url, 'published': published})[0][0] - if exists: + threshold = Config.lastpubmsg + self.__postdelay + now = time.time() + if now >= threshold: + """Checks if a news item with the given information exists. If not, create a new entry.""" + exists = self.__db_worker.execute("select exists(select 1 FROM news WHERE feedid = :feedid and url = :url and published = :published LIMIT 1)", {'feedid': feed_id, 'url': url, 'published': published})[0][0] + if exists: + return False + self.__db_worker.execute("INSERT INTO news (title, url, feedid, published) VALUES (:title, :url, :feedid, :published)", {'title': title, 'url': url, 'feedid': feed_id, 'published': published}) + return True + else: return False - self.__db_worker.execute("INSERT INTO news (title, url, feedid, published) VALUES (:title, :url, :feedid, :published)", {'title': title, 'url': url, 'feedid': feed_id, 'published': published}) - return True diff --git a/feedupdater.py b/feedupdater.py index db7a6c9..fe01190 100644 --- a/feedupdater.py +++ b/feedupdater.py @@ -1,13 +1,14 @@ -#!/usr/bin/python2.7 +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- import feedparser import datetime import dateutil.parser import signal import time -import tinyurl import threading import os +import sys from db import FeedDB from config import Config @@ -49,12 +50,8 @@ def __fetch_feed(self, feed_info, callback, forever): # Reverse the ordering. Oldest first. for newsitem in news.entries[::-1]: newstitle = newsitem.title - if self.__config.shorturls: - newsurl = tinyurl.create_one(newsitem.link) # Create a short link - if newsurl == "Error": #If that fails, use the long version - newsurl = newsitem.link - else: - newsurl = newsitem.link + newsurl = newsitem.link +# print datetime.datetime.now(), newsurl # Try to get the published or updated date. Otherwise set it to 'no date' try: @@ -71,16 +68,17 @@ def __fetch_feed(self, feed_info, callback, forever): newsdate = newsdate.strftime(self.__config.dateformat) except Exception as e: - newsdate = "no date" + newsdate = u"No date" # Update the database. If it's a new issue, post it to the channel is_new = self.__db.insert_news(feed_info['id'], newstitle, newsitem.link, newsdate) if is_new and callback is not None: callback(feed_info['title'], newstitle, newsurl, newsdate) - print "Updated: " + feed_info['title'] except Exception as e: - print e - print "Failed: " + feed_info['title'] + print datetime.datetime.now(), e + print datetime.datetime.now(), u"Feed not updated: " + feed_info['title'] + sys.stdout.flush() + if not forever: break @@ -90,18 +88,22 @@ def __fetch_feed(self, feed_info, callback, forever): if __name__ == "__main__": def print_line(feed_title, news_title, news_url, news_date): - print(u"[+]: {}||{}||{}||{}".format(feed_title, news_title, news_url, news_date)) + print datetime.datetime.now(), u"[+]: {}||{}||{}||{}".format(feed_title, news_title, news_url, news_date) + sys.stdout.flush() def main(): config = Config() db = FeedDB(config) updater = FeedUpdater(config, db) + print datetime.datetime.now(), u"Starting offline update." + sys.stdout.flush() updater.update_feeds(print_line, False) def signal_handler(signal, frame): - print "Caught SIGINT, terminating." + print datetime.datetime.now(), u"Received SIGINT signal, finishing bot." + sys.stdout.flush() os._exit(0) signal.signal(signal.SIGINT, signal_handler) - main() \ No newline at end of file + main() diff --git a/main.py b/main.py index 3c173f0..f720e68 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,22 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- + from bot import Bot from feedupdater import FeedUpdater +from config import Config import os import signal +import datetime +import sys +import codecs +import locale + +sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout) +sys.stderr = codecs.getwriter(locale.getpreferredencoding())(sys.stderr) def signal_handler(signal, frame): - print "Caught SIGINT, terminating." + print datetime.datetime.now() , u"Received SIGINT signal, finishing bot." + sys.stdout.flush() os._exit(0) if __name__ == "__main__": @@ -15,7 +25,8 @@ def signal_handler(signal, frame): missing_config_keys = bot.get_missing_options() if not len(missing_config_keys) == 0: for key in missing_config_keys: - print "Config option '{}' is missing! Please check your config!".format(key) + print datetime.datetime.now(), u"The '{}' option isn't set up! Check configuration.".format(key) + sys.stdout.flush() os._exit(1) bot._Bot__irc.connection.buffer_class.errors = 'replace' # prevent utf-8 error in jaraco.stream