Skip to content

Commit

Permalink
add request and response headers to template
Browse files Browse the repository at this point in the history
  • Loading branch information
@ninapavlich
ninapavlich committed Jul 17, 2015
1 parent 6c46407 commit f57865f
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 24 deletions.
36 changes: 19 additions & 17 deletions sitetest/core/models.py
View file
Expand Up @@ -55,7 +55,7 @@
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}

USE_REQUESTS = True
USE_REQUESTS = False

import ssl
from functools import wraps
Expand Down Expand Up @@ -545,7 +545,7 @@ def __init__(self, url, set, verbose=False):
self.iframe_links = {}
self.screenshots = {}
# self.xhr_links = {}
self.url = self.ending_url = url
self.url = self.starting_url = self.ending_url = url
parsed = urlparse.urlparse(url)
name, extension = os.path.splitext(parsed.path)
self.starting_type = self.ending_type = set.get_link_type(url)
Expand All @@ -558,6 +558,7 @@ def __init__(self, url, set, verbose=False):
self.html = None
self.title = url
self.redirect_path = None
self.has_sitemap_entry = False
self.dequeried_url = clear_query_string(self.url)

self.use_basic_auth = set.use_basic_auth
Expand Down Expand Up @@ -701,7 +702,7 @@ def is_200(self):

@property
def is_redirect_page(self):
return (self.url != self.ending_url)
return (self.starting_url != self.ending_url)



Expand Down Expand Up @@ -1190,7 +1191,7 @@ def trace_path_with_requests(url, is_internal, traced, enable_cookies = False, d
#-- authorization

#Don't verify cert here if we're testing the site. We'll test that on a separate step.
verify_cert = !is_internal
verify_cert = not is_internal

if session:
response = session.get(url, headers=HEADERS, allow_redirects=False, verify=verify_cert, timeout=10)
Expand Down Expand Up @@ -1235,7 +1236,7 @@ def trace_path_with_requests(url, is_internal, traced, enable_cookies = False, d
if enable_cookies:
response_data['pickled_cookies'] = pickle.dumps(session.cookies._cookies)

response_data['request_headers'] = response.request.headers


traced.append(response_data)
Expand Down Expand Up @@ -1277,34 +1278,31 @@ def trace_path_with_urllib2(url, is_internal, traced, enable_cookies = False, de
traced[-1]['error'] = "Redirect loop detected to %s"%(url)
return traced


if enable_cookies:
use_auth = (auth=='basic' and is_internal)
if enable_cookies or use_auth:
if not cj:
cj = cookielib.CookieJar()

cj = cookielib.CookieJar()

use_password = False
if auth=='basic' and is_internal:
use_password = True
if use_auth:
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, url, username, password)
password_handler = urllib2.HTTPBasicAuthHandler(password_manager)


if enable_cookies:
if use_password:
if cj:
if use_auth:
opener = urllib2.build_opener(NoRedirection, urllib2.HTTPCookieProcessor(cj), password_handler)
else:
opener = urllib2.build_opener(NoRedirection, urllib2.HTTPCookieProcessor(cj))
else:

if use_password:

if use_auth:
opener = urllib2.build_opener(NoRedirection, password_handler)
else:
opener = urllib2.build_opener(NoRedirection)

request = urllib2.Request(url, headers=HEADERS)


response = None
response_data = {
Expand Down Expand Up @@ -1375,6 +1373,8 @@ def trace_path_with_urllib2(url, is_internal, traced, enable_cookies = False, de
if enable_cookies:
response_data['pickled_cookies'] = pickle.dumps(cj._cookies)

response_data['request_headers'] = request.headers

traced.append(response_data)

has_redirect = response_data['redirect']!=None
Expand All @@ -1397,6 +1397,7 @@ def parse_trace_response(response, response_data, code, response_header, start_t
def parse_trace_response_with_requests(response, response_data, code, response_header, start_time):

end_time = datetime.datetime.now()
response_data['response_headers'] = response_header
response_data['response_code'] = code
response_data['response_content_type'] = response_header.get('Content-Type')
response_data['response_encoding'] = response_header.get('Content-Encoding')
Expand All @@ -1415,8 +1416,9 @@ def parse_trace_response_with_requests(response, response_data, code, response_h
response_data['redirect'] = response_data['ending_url']

def parse_trace_response_with_urllib2(response, response_data, code, response_header, start_time):

end_time = datetime.datetime.now()
response_data['response_headers'] = dict(response_header)
response_data['response_code'] = code
response_data['response_content_type'] = response_header.getheader('Content-Type')
response_data['response_encoding'] = response_header.getheader('Content-Encoding')
Expand Down
3 changes: 2 additions & 1 deletion sitetest/core/sitemap.py
View file
Expand Up @@ -59,6 +59,7 @@ def process_sitemap(self, sitemap_link):
if not sitemap_link.content:
return


xml = sitemap_link.content.encode('utf-8')
parser = lxml.etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8')
tree = lxml.etree.fromstring(xml, parser=parser)
Expand All @@ -75,6 +76,6 @@ def process_sitemap(self, sitemap_link):

url = loc.text.strip()
self.urls.append(url)
# print 'Found url %s'%(url)
# print 'Found url (%s)'%(url)
link = self.set.get_or_create_link_object(url, sitemap_link)

13 changes: 8 additions & 5 deletions sitetest/templates/partials/result_title.html
View file
@@ -1,20 +1,23 @@
<h4 class="panel-title clearfix" id="{{link.page_hash}}">
<span class="pull-left">

{% if link.response_code %}
{% if link.response_code %}
{% if link.response_code == 200 %}
<span class="label label-success">{{link.response_code}}</span>
{% elif link.response_code == "Unknown" %}
<span class="label label-warning">Response Code {{link.response_code}}</span>
{% else %}
<span class= "label label-danger">{{link.response_code}}</span>
{% endif %}
{% else %}
{% else %}
<span class="label label-default">N/A</span>
{% endif %}
{% endif %}

{% if link.is_redirect_page %}&nbsp;<span class="label label-default">Redirected</span> {% endif %}

&nbsp;
<a data-parent="#accordion" data-toggle="collapse" href="#{{link.page_results_hash}}">{% if link.is_redirect_page %}<span class="label label-default">Redirected</span> {% endif %}{% if link.title %}{{link.title}}<br /><br />{% endif %}
<span class='link-container'>{{link.url|truncate(100, True)}}{% if link.url != link.ending_url %} > {{link.ending_url|truncate(100, True)}} {% endif %}</span></a>
<a data-parent="#accordion" data-toggle="collapse" href="#{{link.page_results_hash}}">{% if link.title %}{{link.title}}<br /><br />{% endif %}
<span class='link-container'>{{link.starting_url|truncate(100, True)}}{% if link.starting_url != link.ending_url %} > {{link.ending_url|truncate(100, True)}} {% endif %}</span></a>


</span>
Expand Down
31 changes: 31 additions & 0 deletions sitetest/templates/result_full.html
View file
Expand Up @@ -160,6 +160,37 @@ <h5>There are {{link.script_links|length}} javascript files linked in this page<

{% if link.enumerated_source %}
<div class="tab-pane" id="{{link.page_results_hash}}-source">


{% for item in link.redirect_path %}

<h4>{{item.response_code}} {{item.url}} [{{item.response_content_type}}] - <em>{{item.response_load_time}} ms</em></h4>
<div class="box">
<div class="inner">
<h5>Request Headers</h5>
<dl>
{% for key, value in item.request_headers.iteritems() %}
<dt>{{ key }}</dt>
<dd>{{ value }}</dd>
{% endfor %}
</dl>
</div>
<div class="inner">
<h5>Response Headers</h5>
<dl>
{% for key, value in item.response_headers.iteritems() %}
<dt>{{ key }}</dt>
<dd>{{ value }}</dd>
{% endfor %}
</dl>
</div>
</div>

<hr />

{% endfor %}


<pre>{{link.enumerated_source|e}}</pre>
</div>
{% endif %}
Expand Down
39 changes: 39 additions & 0 deletions sitetest/templates/results_base.html
View file
Expand Up @@ -98,6 +98,45 @@
font-weight: normal;
}

.box{
width:100%;
overflow:hidden;
}
.box .inner{
width:50%;
float:left;
padding-right: 20px;
}
.box .inner:last-child{
padding-right:0;
}
dl {
display: block;
padding: 10px;
margin: 0 0 10px;
color: #333;
word-break: break-all;
word-wrap: break-word;
background-color: #f5f5f5;
border: 1px solid #ccc;
border-radius: 4px;
}
dt {
float: left;
clear: left;
width: 25%;
margin-right:7px;
text-align: right;
font-weight: bold;
}
dt:after {
content: ":";
}
dd {
padding: 0 0 0.5em 0;
border-bottom: 1px solid #fff;
}


</style>

Expand Down
2 changes: 1 addition & 1 deletion sitetest/tests/basic_site_quality.py
View file
Expand Up @@ -52,13 +52,13 @@ def test_basic_site_quality(site, verbose=False):


# #5 - Verify that sitemap matches up with the actual pages
# if sitemap_link.response_code == 200:
if len(sitemap_links) > 0:
for sitemap_link in sitemap_links:
for link_url in sitemap_link.hyper_links:
link_item = sitemap_link.hyper_links[link_url]
link_item.has_sitemap_entry = True



# #6 - Verify that no public pages are blocked by robots.txt
rp = robotparser.RobotFileParser()
Expand Down
4 changes: 4 additions & 0 deletions sitetest/tests/pagespeed.py
View file
Expand Up @@ -42,8 +42,12 @@ def test_pagespeed(set, credentials, options, max_test_count=1000, verbose=False
else:
url = link.url



testing_url = 'https://www.googleapis.com/pagespeedonline/v1/runPagespeed?url=%s&key=%s'%(url, API_KEY)

# print testing_url

request = urllib2.Request(testing_url)
response = urllib2.urlopen(request)

Expand Down

0 comments on commit f57865f

Please sign in to comment.