Skip to content
This repository has been archived by the owner on Jan 10, 2019. It is now read-only.

Commit

Permalink
PEP8
Browse files Browse the repository at this point in the history
  • Loading branch information
@desbma
desbma committed Oct 18, 2012
1 parent 2c8afe6 commit 2093e1d
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 89 deletions.
53 changes: 26 additions & 27 deletions ddc_client.py
View file
Expand Up @@ -11,7 +11,7 @@ class DebugLogRecordFactory():
def __init__(self): def __init__(self):
self.default_logrecord_factory = logging.getLogRecordFactory() self.default_logrecord_factory = logging.getLogRecordFactory()


def log(self,*args, **kwargs): def log(self, *args, **kwargs):
record = self.default_logrecord_factory(*args, **kwargs) record = self.default_logrecord_factory(*args, **kwargs)
record.msg = "[CLIENT] %s" % (record.msg) record.msg = "[CLIENT] %s" % (record.msg)
return record return record
Expand All @@ -23,7 +23,7 @@ class NeedRestartException(Exception):


class InvalidServerResponse(Exception): class InvalidServerResponse(Exception):


def __init__(self,http_code): def __init__(self, http_code):
self.http_code = http_code self.http_code = http_code


def __str__(self): def __str__(self):
Expand All @@ -35,8 +35,8 @@ class DistributedCrawlerClient():
CLIENT_VERSION = 1 CLIENT_VERSION = 1
http_client = httplib2.Http(timeout=10) http_client = httplib2.Http(timeout=10)


def __init__(self,server,port): def __init__(self, server, port):
self.base_url = "http://%s:%d" % (server,port) self.base_url = "http://%s:%d" % (server, port)
self.api_base_url = "%s/domains" % (self.base_url) self.api_base_url = "%s/domains" % (self.base_url)


def start(self,): def start(self,):
Expand All @@ -48,8 +48,8 @@ def start(self,):


try: try:
# see README.md for params description # see README.md for params description
response = self.apiRequest({ "version" : str(__class__.CLIENT_VERSION), response = self.apiRequest({"version" : str(__class__.CLIENT_VERSION),
"pc_version" : str(ddc_process.VERSION) }).decode("utf-8") "pc_version" : str(ddc_process.VERSION)}).decode("utf-8")


# read response # read response
xml_response = xml.etree.ElementTree.fromstring(response) xml_response = xml.etree.ElementTree.fromstring(response)
Expand All @@ -61,13 +61,13 @@ def start(self,):
for xml_upgrade in xml_response.findall("upgrades/upgrade"): for xml_upgrade in xml_response.findall("upgrades/upgrade"):
type = xml_upgrade.get("type") type = xml_upgrade.get("type")
version = xml_upgrade.get("version") version = xml_upgrade.get("version")
logging.getLogger().info("Upgrading '%s' component to version %s" % (type,version) ) logging.getLogger().info("Upgrading '%s' component to version %s" % (type, version))
url = self.base_url + xml_upgrade.get("url") url = self.base_url + xml_upgrade.get("url")
response, content = __class__.http_client.request(url) response, content = __class__.http_client.request(url)
zip_filename = response["content-disposition"].split(";")[1].split("=")[1] zip_filename = response["content-disposition"].split(";")[1].split("=")[1]
with open(zip_filename,"r+b") as file_handle: with open(zip_filename, "r+b") as file_handle:
file_handle.write(content) file_handle.write(content)
archive = zipfile.ZipFile(file_handle,"r") archive = zipfile.ZipFile(file_handle, "r")
archive.extractall() archive.extractall()
archive.close() archive.close()
need_restart = True need_restart = True
Expand All @@ -81,12 +81,12 @@ def start(self,):
continue continue


# check domains # check domains
logging.getLogger().info("Got %d domains to check from server" % (domain_count) ) logging.getLogger().info("Got %d domains to check from server" % (domain_count))
spam_domain_indexes = set() spam_domain_indexes = set()
failed_domain_indexes = set() failed_domain_indexes = set()
for (i, xml_domain) in enumerate(xml_domains): for (i, xml_domain) in enumerate(xml_domains):
domain = xml_domain.get("name") domain = xml_domain.get("name")
logging.getLogger().debug("Checking domain '%s'" % (domain) ) logging.getLogger().debug("Checking domain '%s'" % (domain))
try: try:
if ddc_process.is_spam(domain): if ddc_process.is_spam(domain):
spam_domain_indexes.add(i) spam_domain_indexes.add(i)
Expand All @@ -95,21 +95,21 @@ def start(self,):


# prepare POST request content # prepare POST request content
xml_root = xml.etree.ElementTree.Element("ddc") xml_root = xml.etree.ElementTree.Element("ddc")
xml_domain_list = xml_response.find("domainlist") # reuse the previous XML domain list xml_domain_list = xml_response.find("domainlist") # reuse the previous XML domain list
for (i, xml_domain) in enumerate(xml_domain_list.iterfind("domain")): for (i, xml_domain) in enumerate(xml_domain_list.iterfind("domain")):
if i in failed_domain_indexes: if i in failed_domain_indexes:
xml_domain.set("failed","1") xml_domain.set("failed", "1")
else: else:
is_spam = (i in spam_domain_indexes) is_spam = (i in spam_domain_indexes)
xml_domain.set("spam",str(int(is_spam))) xml_domain.set("spam", str(int(is_spam)))
xml_root.append(xml_domain_list) xml_root.append(xml_domain_list)


# send POST request # send POST request
post_data = xml.etree.ElementTree.tostring(xml_root) post_data = xml.etree.ElementTree.tostring(xml_root)
self.apiRequest( { "version" : str(__class__.CLIENT_VERSION), self.apiRequest({"version" : str(__class__.CLIENT_VERSION),
"pc_version" : str(ddc_process.VERSION) }, "pc_version" : str(ddc_process.VERSION)},
True, True,
post_data) # we don't care for what the server actually returns here post_data) # we don't care for what the server actually returns here


except InvalidServerResponse as e: except InvalidServerResponse as e:
logging.getLogger().warning(e) logging.getLogger().warning(e)
Expand All @@ -118,16 +118,15 @@ def start(self,):
logging.getLogger().info("Restarting client") logging.getLogger().info("Restarting client")
exit(7) exit(7)



def apiRequest(self, url_params, post_request=False, post_data=None):
def apiRequest(self,url_params,post_request=False,post_data=None):
# construct url # construct url
url = "%s?%s" % (self.api_base_url,urllib.parse.urlencode(url_params)) url = "%s?%s" % (self.api_base_url, urllib.parse.urlencode(url_params))
# send request # send request
if post_request: if post_request:
logging.getLogger().info("Posting data to '%s'" % (url) ) logging.getLogger().info("Posting data to '%s'" % (url))
response, content = __class__.http_client.request(url,"POST",post_data) response, content = __class__.http_client.request(url, "POST", post_data)
else: else:
logging.getLogger().info("Fetching '%s'" % (url) ) logging.getLogger().info("Fetching '%s'" % (url))
response, content = __class__.http_client.request(url) response, content = __class__.http_client.request(url)
if response.status not in (200, 202): if response.status not in (200, 202):
raise InvalidServerResponse(response.status) raise InvalidServerResponse(response.status)
Expand All @@ -154,7 +153,7 @@ def apiRequest(self,url_params,post_request=False,post_data=None):
cli_parser.add_argument("-v", cli_parser.add_argument("-v",
"--verbosity", "--verbosity",
action="store", action="store",
choices=("quiet","warning","info","debug"), choices=("quiet", "warning", "info", "debug"),
default="info", default="info",
dest="verbosity", dest="verbosity",
help="Level of output to diplay") help="Level of output to diplay")
Expand All @@ -164,7 +163,7 @@ def apiRequest(self,url_params,post_request=False,post_data=None):
logging.basicConfig(format="%(message)s") logging.basicConfig(format="%(message)s")
logger = logging.getLogger() logger = logging.getLogger()
if options.verbosity == "quiet": if options.verbosity == "quiet":
logger.setLevel(logging.CRITICAL+1) logger.setLevel(logging.CRITICAL + 1)
elif options.verbosity == "warning": elif options.verbosity == "warning":
logger.setLevel(logging.WARNING) logger.setLevel(logging.WARNING)
elif options.verbosity == "info": elif options.verbosity == "info":
Expand All @@ -175,5 +174,5 @@ def apiRequest(self,url_params,post_request=False,post_data=None):
logging.setLogRecordFactory(logrecord_factory.log) logging.setLogRecordFactory(logrecord_factory.log)


# start client # start client
client = DistributedCrawlerClient(options.server,options.port) client = DistributedCrawlerClient(options.server, options.port)
client.start() client.start()

0 comments on commit 2093e1d

Please sign in to comment.