From bc63a1caa48a542df4fa0e877ede362c64ddcd25 Mon Sep 17 00:00:00 2001 From: odie5533 Date: Fri, 25 Oct 2013 03:55:46 -0500 Subject: [PATCH] Refactor warcmiddleware.py --- warcmiddleware.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/warcmiddleware.py b/warcmiddleware.py index e401d6c..46ecc50 100644 --- a/warcmiddleware.py +++ b/warcmiddleware.py @@ -1,9 +1,9 @@ import urlparse from cStringIO import StringIO -import scrapy.http import twisted.web.http from scrapy.utils.httpobj import urlparse_cached +import scrapy.http import warcrecords @@ -34,7 +34,7 @@ def __init__(self): Follows most of the code from scrapy/core/downloader/webclient.py """ - def warcrec_from_scrapy_request(self, request): + def _warcrec_from_scrapy_request(self, request): headers = request.headers body = request.body @@ -66,7 +66,7 @@ def warcrec_from_scrapy_request(self, request): tofix: Handle response.status codes """ - def warcrec_from_scrapy_response(self, response): + def _warcrec_from_scrapy_response(self, response): # Everything is OK. resp_str = "HTTP/1.0 " + str(response.status) + " OK\r\n" resp_str += response.headers.to_string() @@ -76,10 +76,10 @@ def warcrec_from_scrapy_response(self, response): return warcrecords.WarcResponseRecord(url=response.url, block=resp_str) def process_request(self, request, spider): - record = self.warcrec_from_scrapy_request(request) + record = self._warcrec_from_scrapy_request(request) record.write_to(self.fo) def process_response(self, request, response, spider): - record = self.warcrec_from_scrapy_response(response) + record = self._warcrec_from_scrapy_response(response) record.write_to(self.fo) return response # return the response to Scrapy for further handling