From 8ae130f3bff622f3f7d356dae32f03c0ea6678ec Mon Sep 17 00:00:00 2001 From: d1ffuz0r Date: Sun, 25 Dec 2011 11:34:55 +0300 Subject: [PATCH] refactoring, --- engine.py | 49 ++++++++------------------------ modules/db.py | 74 +++++++++++++++++++++++++++++++++---------------- tests/parser.py | 4 +-- 3 files changed, 63 insertions(+), 64 deletions(-) diff --git a/engine.py b/engine.py index f9501ba..aae9e74 100644 --- a/engine.py +++ b/engine.py @@ -8,23 +8,22 @@ from re import compile, search, DOTALL, findall from cookielib import CookieJar -from modules.db import Player, Clan, Online, \ - InOnline, Profession +from modules.db import Online, _online_now, add class Parser(object): + """Parser class. + To parse and save in storage information from html pages + """ - """Parser class. To parse and save in storage information from html pages""" - - def __init__(self, pause=5): - """Initialization class - - Keyword arguments: - pause -- pause between getting pages, in minutes (default 5) + def __init__(self, server='x5', pause=5): + """ + Initialization class + @param pause: pause between getting pages, in minutes (default 5) """ #todo to realize work with many servers (asterios, l2, theonline) - self.url = "http://www.l2planet.ws/?go=online&server=x5" + self.url = "http://www.l2planet.ws/?go=online&server=%s" % server self.regexp = compile(r'.*(.*?)', DOTALL) self.regexp2 = compile(r'.*?(?P.*?).*?.*?(?P.*?)(?P.*?).*?') self.pause = pause @@ -38,40 +37,14 @@ def get_content(self): page = urlopen(Request(self.url, headers=self.headers)) return page.read() - def add(self, name, profa, clan): - """Get or create information in database - - Keyword arguments: - name -- name user - profa -- profession name - clan -- clan name - - """ - clan = Clan.get_or_create(name=clan) - profa = Profession.get_or_create(name=profa) - name = Player.get_or_create(name=name, profession=profa, clan=clan) - logging.warning({"clan": clan, "profa": profa, "name": name}) - return {"clan": clan, "profa": profa, "name": name} - - def _online_now(self, data): - """To append all users who in online now - - Keyword arguments: - data -- all players in online - - """ - online_now = data['time'] - for player in data["online"]: - InOnline.create(player=player["name"], online=online_now) - def parse(self): """Parse page""" time = Online().create(date=datetime.datetime.now()) res = {"time": time, "online": []} html = search(self.regexp, self.get_content()).group(1).replace('\r\n', '') for line in findall(self.regexp2, html): - res["online"].append(self.add(line[0], line[1], line[2])) - self._online_now(res) + res["online"].append(add(line[0], line[1], line[2])) + _online_now(res) def start(self): """Start engine""" diff --git a/modules/db.py b/modules/db.py index 7a703f7..45fb27f 100644 --- a/modules/db.py +++ b/modules/db.py @@ -1,4 +1,5 @@ from datetime import datetime +import logging from config import conf from modules.peewee import MySQLDatabase, Model, CharField,\ @@ -19,10 +20,10 @@ class Meta: class Profession(BaseModel): - """Profession model. To store profession names + """ + Profession model. To store profession names - Keyword arguments: - name -- profession name + @param name: name of profession tablename: profession """ @@ -36,10 +37,10 @@ def __unicode__(self): class Clan(BaseModel): - """Clan model. To store clans names + """ + Clan model. To store clans names - Keyword arguments: - name -- clan name + @param name: name of clan tablename: clan """ @@ -53,10 +54,10 @@ def __unicode__(self): class Online(BaseModel): - """Online model. To store online stamp + """ + Online model. To store online stamp - Keyword arguments: - date -- datetime stamp + @param date: timeshtamp when make parse tablename: online """ @@ -70,12 +71,12 @@ def __unicode__(self): class Player(BaseModel): - """Player model. To store player information's + """ + Player model. To store player information's - Keyword arguments: - name -- player nmame - profession -- profession id - clan -- clan id + @param name: name of player + @param profession: profession id + @param clan: clan id tablename: player """ @@ -91,11 +92,11 @@ def __unicode__(self): class InOnline(BaseModel): - """InOnline model. To store relations between Player and Online models + """ + InOnline model. To store relations between Player and Online models - Keyword arguments: - player -- player id - online -- inline id + @param player: player id + @param online: online id tablename: online_players """ @@ -124,13 +125,13 @@ def create_tables(): def get_by_nick(nick=None, frm=None, to=None): - """To get order by nick - - Keyword arguments: - nick -- player name - frm -- filter date from - to -- filter date to + """ + To get order by nick + @param nick: player name + @param frm: filter date from + @param to: filter date to + @return dict: dates and player state """ if nick: player = Player.select().where(name=nick).get() @@ -168,3 +169,28 @@ def get_by_nick(nick=None, frm=None, to=None): return {'all': dates, 'player': all} else: return None + +def add(name, profa, clan): + """ + Get or create information in database + + @param name: name user + @param profa: profession name + @param clan: clan name + @return dict: + """ + clan = Clan.get_or_create(name=clan) + profa = Profession.get_or_create(name=profa) + name = Player.get_or_create(name=name, profession=profa, clan=clan) + logging.warning({"clan": clan, "profa": profa, "name": name}) + return {"clan": clan, "profa": profa, "name": name} + +def _online_now(data): + """ + To append all users who in online now + + @param data: all players in online + """ + online_now = data['time'] + for player in data["online"]: + InOnline.create(player=player["name"], online=online_now) diff --git a/tests/parser.py b/tests/parser.py index 7f26ef8..73d4d3c 100644 --- a/tests/parser.py +++ b/tests/parser.py @@ -9,7 +9,7 @@ def setUp(self): self.parser1 = Parser("x5") self.parser2 = Parser("x1") self.parser = Parser("x5") - self.parser.url = "file:///F:/x5.htm" + self.parser.url = "file:///D:/C/x5.htm" def test_create(self): self.assertIsNotNone(self.parser) @@ -19,7 +19,7 @@ def test_params(self): "http://www.l2planet.ws/?go=online&server=x5") self.assertEqual(self.parser2.url, "http://www.l2planet.ws/?go=online&server=x1") - self.assertEqual(self.parser.pause, 0) + self.assertEqual(self.parser.pause, 5) self.assertEqual(self.parser.headers, {"User-Agent": "Mozilla/4.0 (compatible; MSIE 7; WindowsNT)"})