Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

switched from event_log to standard python logging

  • Loading branch information...
commit 4b4530ffd9017c4cbf451f4b008831adb3f67d07 1 parent a3a8b1f
Bernhard Haslhofer authored
View
11 config/default.yaml
@@ -39,13 +39,4 @@ changememory:
# jid: publisher_jid
# pwd: pwd
# pubsub_node: node_name
-# pubsub_jid: pubsub.host
-
-###### Event logger implementations ######
-
-logger:
- class: ConsoleEventLog
-
-# logger:
-# class: FileEventLog
-# filename: source_events.log
+# pubsub_jid: pubsub.host
View
3  resync/changememory.py
@@ -6,6 +6,7 @@
Created by Bernhard Haslhofer on 2012-04-27.
Copyright 2012, ResourceSync.org. All rights reserved.
"""
+import logging
from resync.observer import Observer
from resync.changeset import ChangeSet
@@ -18,6 +19,7 @@ class ChangeMemory(Observer):
def __init__(self, source):
self.source = source
source.register_observer(self)
+ self.logger = logging.getLogger('changememory')
def bootstrap(self):
"""Bootstrap the Changememory; should be overridden by subclasses"""
@@ -62,6 +64,7 @@ def generate(self, from_changeid=None):
def notify(self, change):
"""Simply store a change in the in-memory list"""
+ self.logger.info(str(change))
change.changeid = self.latest_change_id + 1
self.latest_change_id = change.changeid
if self.max_changes != -1 and self.change_count >= self.max_changes:
View
39 resync/event_log.py
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-"""
-event_log.py: A collection of event logger implementations
-
-Created by Bernhard Haslhofer on 2012-04-24.
-Copyright 2012, ResourceSync.org. All rights reserved.
-"""
-
-import logging
-
-from observer import Observer
-
-class ConsoleEventLog(Observer):
- """This EventLog logs change events to the console"""
-
- def __init__(self, observable, config=None):
- observable.register_observer(self)
-
- def notify(self, event):
- print "Event: " + str(event)
-
-class FileEventLog(Observer):
- """This EventLog logs change events to a log file"""
-
- def __init__(self, observable, config=None):
- observable.register_observer(self)
- if config is None:
- logfile = "events.log"
- else:
- logfile = config['filename']
- logging.basicConfig(filename=logfile,
- format='%(asctime)s %(message)s',
- datefmt='%Y-%m-%dT%H:%M:%S',
- level=logging.INFO)
-
- def notify(self, event):
- logging.info(str(event))
-
View
6 resync/http.py
@@ -10,6 +10,7 @@
import threading
import os.path
+import logging
import tornado.httpserver
import tornado.ioloop
@@ -34,6 +35,7 @@ class HTTPInterface(threading.Thread):
def __init__(self, source):
"""Initializes HTTP interface with default settings and handlers"""
super(HTTPInterface, self).__init__()
+ self.logger = logging.getLogger('http')
self._stop = threading.Event()
self.source = source
self.port = source.port
@@ -77,7 +79,7 @@ def __init__(self, source):
def run(self):
- print "*** Starting up HTTP Interface on port %i ***\n" % (self.port)
+ self.logger.info("Starting up HTTP Interface on port %i" % (self.port))
application = tornado.web.Application(
handlers = self.handlers,
debug = True,
@@ -87,7 +89,7 @@ def run(self):
tornado.ioloop.IOLoop.instance().start()
def stop(self):
- print "*** Stopping HTTP Interface ***\n"
+ self.logger.info("Stopping HTTP Interface")
tornado.ioloop.IOLoop.instance().stop()
self._stop.set()
View
20 resync/sitemap.py
@@ -3,6 +3,7 @@
import re
import os
import sys
+import logging
from urllib import URLopener
from xml.etree.ElementTree import ElementTree, Element, parse, tostring
from datetime import datetime
@@ -43,6 +44,7 @@ class Sitemap(object):
def __init__(self, verbose=False, pretty_xml=False, allow_multifile=True,
mapper=None):
+ self.logger = logging.getLogger('sitemap')
self.verbose=verbose
self.pretty_xml=pretty_xml
self.allow_multifile=allow_multifile
@@ -85,26 +87,26 @@ def write(self, resources=None, basename='/tmp/sitemap.xml'):
for i in range(0,len(resources),self.max_sitemap_entries):
file = sitemap_prefix + ( "%05d" % (len(sitemaps)) ) + sitemap_suffix
if (self.verbose):
- print "Writing sitemap %s..." % (file)
+ self.logger.info("Writing sitemap %s..." % (file))
f = open(file, 'w')
f.write(self.resources_as_xml(resources,num_resources=self.max_sitemap_entries,include_capabilities=False))
f.close()
# Record timestamp
sitemaps[file] = os.stat(file).st_mtime
- print "Wrote %d sitemaps" % (len(sitemaps))
+ self.logger.info("Wrote %d sitemaps" % (len(sitemaps)))
f = open(basename, 'w')
if (self.verbose):
- print "Writing sitemapindex %s..." % (basename)
+ self.logger.info("Writing sitemapindex %s..." % (basename))
f.write(self.sitemapindex_as_xml(sitemaps=sitemaps,inventory=resources,include_capabilities=True))
f.close()
- print "Wrote sitemapindex %s" % (basename)
+ self.logger.info("Wrote sitemapindex %s" % (basename))
else:
f = open(basename, 'w')
if (self.verbose):
- print "Writing sitemap %s..." % (basename)
+ self.logger.info("Writing sitemap %s..." % (basename))
f.write(self.resources_as_xml(resources))
f.close()
- print "Wrote sitemap %s" % (basename)
+ self.logger.info("Wrote sitemap %s" % (basename))
def read(self, uri=None, resources=None):
"""Read sitemap from a URI including handling sitemapindexes
@@ -150,7 +152,6 @@ def read(self, uri=None, resources=None):
raise Exception("Failed to load sitemap from %s listed in sitemap index %s (%s)" % (sitemap_uri,uri,str(e)))
self.inventory_parse_xml( fh=fh, inventory=resources )
self.sitemaps_created+=1
- #print "%s : now have %d resources" % (sitemap_uri,len(resources))
else:
raise ValueError("XML is not sitemap or sitemapindex")
return(resources)
@@ -305,7 +306,8 @@ def inventory_parse_xml(self, fh=None, etree=None, inventory=None):
try:
inventory.add( r )
except InventoryDupeError:
- print "dupe: %s (%s =? %s)" % (r.uri,r.lastmod,inventory.resources[r.uri].lastmod)
+ self.logger.warning("dupe: %s (%s =? %s)" %
+ (r.uri,r.lastmod,inventory.resources[r.uri].lastmod))
self.resources_created+=1
return(inventory)
elif (etree.getroot().tag == '{'+SITEMAP_NS+"}sitemapindex"):
@@ -342,7 +344,7 @@ def sitemapindex_as_xml(self, file=None, sitemaps={}, inventory=None, include_ca
except MapperError:
uri = 'file://'+file
if (self.verbose):
- print "sitemapindex: can't map %s into URI space, writing %s" % (file,uri)
+ self.logger.error("sitemapindex: can't map %s into URI space, writing %s" % (file,uri))
# Make a Resource for the Sitemap and serialize
smr = Resource( uri=uri, timestamp=sitemaps[file] )
root.append( self.resource_etree_element( smr ) )
View
36 resync/source.py
@@ -35,6 +35,7 @@ class DynamicInventoryBuilder(object):
def __init__(self, source, config):
self.source = source
self.config = config
+ self.logger = logging.getLogger('inventory_builder')
def bootstrap(self):
"""Bootstrapping procedures implemented in subclasses"""
@@ -52,14 +53,14 @@ def uri(self):
def generate(self):
"""Generates an inventory (snapshot from the source)"""
+ self.logger.debug("Start inventory generation")
capabilities = {}
if self.source.has_changememory:
next_changeset = self.source.changememory.next_changeset_uri()
capabilities[next_changeset] = {"type": "changeset"}
inventory = Inventory(resources=self.source.resources,
capabilities=capabilities)
- # for resource in self.source.resources:
- # if resource is not None: inventory.add(resource)
+ self.logger.debug("Finished inventory generation")
return inventory
class StaticInventoryBuilder(DynamicInventoryBuilder):
@@ -95,13 +96,14 @@ def generate(self):
def write_static_inventory(self):
"""Writes the inventory to the filesystem"""
-
# Log Sitemap create start event
+ self.logger.info("Start writing Sitemap inventory.")
sm_write_start = ResourceChange(
resource = ResourceChange(self.uri,
timestamp=time.time()),
changetype = "SITEMAP UPDATE START")
self.source.notify_observers(sm_write_start)
+
# Generate sitemap in temp directory
self.ensure_temp_dir(Source.TEMP_FILE_PATH)
inventory = self.generate()
@@ -116,6 +118,7 @@ def write_static_inventory(self):
shutil.rmtree(Source.TEMP_FILE_PATH)
# Log Sitemap create start event
sitemap_size = self.compute_sitemap_size(Source.STATIC_FILE_PATH)
+ self.logger.info("Finished writing Sitemap inventory")
sm_write_end = ResourceChange(
resource = ResourceChange(self.uri,
size=sitemap_size,
@@ -141,7 +144,8 @@ def rm_sitemap_files(self, directory):
"""Deletes sitemap files (from previous runs)"""
filelist = self.ls_sitemap_files(directory)
if len(filelist) > 0:
- print "*** Cleaning up %d old sitemap files ***" % len(filelist)
+ self.logger.debug("*** Cleaning up %d sitemap files ***" %
+ len(filelist))
for f in filelist:
filepath = directory + "/" + f
os.remove(filepath)
@@ -150,7 +154,8 @@ def mv_sitemap_files(self, src_directory, dst_directory):
"""Moves sitemaps from src to dst directory"""
filelist = self.ls_sitemap_files(src_directory)
if len(filelist) > 0:
- print "*** Moving %d sitemap files ***" % len(filelist)
+ self.logger.debug("*** Moving %d sitemap files ***" %
+ len(filelist))
for f in filelist:
filepath = src_directory + "/" + f
shutil.move(filepath, dst_directory)
@@ -172,6 +177,7 @@ class Source(Observable):
def __init__(self, config, hostname, port):
"""Initalize the source"""
super(Source, self).__init__()
+ self.logger = logging.getLogger('source')
self.config = config
self.hostname = hostname
self.port = port
@@ -204,10 +210,10 @@ def has_changememory(self):
def bootstrap(self):
"""Bootstrap the source with a set of resources"""
- print "*** Bootstrapping source with %d resources and an average " \
- "resource payload of %d bytes ***" \
+ self.logger.info("Bootstrapping source with %d resources " \
+ "and an average resource payload of %d bytes" \
% (self.config['number_of_resources'],
- self.config['average_payload'])
+ self.config['average_payload']))
for i in range(self.config['number_of_resources']):
self._create_resource(notify_observers = False)
@@ -234,8 +240,8 @@ def resources(self):
for basename in repository.keys():
resource = self.resource(basename)
if resource is None:
- print "Cannot create resource %s " % basename + \
- "because source object has been deleted."
+ self.logger.error("Cannot create resource %s " % basename + \
+ "because source object has been deleted.")
yield resource
@property
@@ -276,9 +282,9 @@ def random_resources(self, number = 1):
def simulate_changes(self):
"""Simulate changing resources in the source"""
- print "*** Starting simulation with change delay %s and event " \
- "types %s ***" \
- % (str(self.config['change_delay']), self.config['event_types'])
+ self.logger.info("Starting simulation with change delay %s " \
+ "and event types %s ***" % (str(self.config['change_delay']),
+ self.config['event_types']))
no_events = 0
sleep_time = self.config['change_delay']
while no_events != self.config['max_events']:
@@ -292,7 +298,6 @@ def simulate_changes(self):
else:
basename = None
if basename is None:
- print "The repository is empty"
no_events = no_events + 1
continue
if event_type == "update":
@@ -301,7 +306,8 @@ def simulate_changes(self):
self._delete_resource(basename)
else:
- print "Event type %s is not supported" % event_type
+ self.logger.error("Event type %s is not supported"
+ % event_type)
no_events = no_events + 1
print "*** Finished change simulation ***"
View
13 simulate-source
@@ -10,11 +10,14 @@ Copyright 2012, ResourceSync.org. All rights reserved.
import argparse
import yaml
+import logging
+import logging.config
from resync.source import Source
from resync.http import HTTPInterface
DEFAULT_CONFIG_FILE = 'config/default.yaml'
+DEFAULT_LOG_FILE = 'config/logging.yaml'
def main():
@@ -24,6 +27,9 @@ def main():
parser.add_argument('--config-file', '-c',
default=DEFAULT_CONFIG_FILE,
help="the simulation configuration file")
+ parser.add_argument('--log-config', '-l',
+ default=DEFAULT_LOG_FILE,
+ help="the logging configuration file")
parser.add_argument('--port', '-p', type=int,
default=8888,
help="the HTTP interface port")
@@ -34,9 +40,12 @@ def main():
# Parse command line arguments
args = parser.parse_args()
+ # Load the logging configuration file and set up logging
+ logconfig = yaml.load(file(args.log_config, 'r'))
+ logging.config.dictConfig(logconfig)
+
# Load the YAML configuration file
- stream = file(args.config_file, 'r')
- config = yaml.load(stream)
+ config = yaml.load(file(args.config_file, 'r'))
# Set up the source
source_settings = config['source']
Please sign in to comment.
Something went wrong with that request. Please try again.