Skip to content

Commit

Permalink
Improve logging during wikipedia parse
Browse files Browse the repository at this point in the history
  • Loading branch information
Greg Nofi committed Mar 8, 2012
1 parent 9bcc62a commit 0a8413e
Showing 1 changed file with 8 additions and 7 deletions.
15 changes: 8 additions & 7 deletions fetch_data/wikiparse.py
Expand Up @@ -15,7 +15,8 @@
import mwclient


logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(message)s')
logger = logging.getLogger("wikiparse")

# Constants
WIKIPEDIA_URL = "en.wikipedia.org"
Expand All @@ -31,10 +32,12 @@

class WikiParse(object):

def __init__(self, wiki_username=None, wiki_password=None):
def __init__(self, wiki_username=None, wiki_password=None, verbose=False):
self.site = mwclient.Site(WIKIPEDIA_URL)
self.site.login(wiki_username, wiki_password)
self.births = rdict()
if verbose:
logger.setLevel(logging.INFO)

def fetch_data(self):
logger.info("Starting Wikipedia parse")
Expand Down Expand Up @@ -110,16 +113,14 @@ def parse_command_line_options():
parser.add_option("--verbose", action="store_true", default=False,
help="Print info messages to stdout")
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("Incorrect number of arguments")
if args:
parser.error("Unknown argument(s): %s" % ", ".join(args))
return options


def main():
options = parse_command_line_options()
if options.verbose:
logger.setLevel(logging.INFO)
w = WikiParse(options.username, options.password)
w = WikiParse(options.username, options.password, options.verbose)
w.fetch_data()
return w.to_json()

Expand Down

0 comments on commit 0a8413e

Please sign in to comment.