Skip to content

Commit

Permalink
Reformats code to better comply with PEP8, especially indentations.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tsvenson
tsvenson committed May 12, 2017
1 parent 850b8c2 commit e7e1f1a
Show file tree
Hide file tree
Showing 5 changed files with 416 additions and 346 deletions.
32 changes: 16 additions & 16 deletions _playground.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,25 @@
import _privatekeys

api = twitter.Api(consumer_key=_privatekeys.twitterConsumerKey,
consumer_secret=_privatekeys.twitterConsumerSecret,
access_token_key=_privatekeys.twitterAccessToken,
access_token_secret=_privatekeys.twitterAccessTokenSecret)
consumer_secret=_privatekeys.twitterConsumerSecret,
access_token_key=_privatekeys.twitterAccessToken,
access_token_secret=_privatekeys.twitterAccessTokenSecret)

print(api.VerifyCredentials())

#statuses = api.GetUserTimeline(screen_name=user)
# statuses = api.GetUserTimeline(screen_name=user)
statuses = api.GetMentions()
for s in statuses:
# behöver tidpunkt, hasha tidpunkt och meddelande till logg över vad som gjorts
#leta upp namnet så det går att svara
# ha hashtag för att skilja ut länkar som ska testas från paper.li och annan skit i mentions
# har BeautifulSoup kod för att hitta URLar?
#print(s.text)
try:
print('Tweet: {0}\nPublished: {1}'.format(s.text, s.created_at))
#{"created_at": "Tue Jun 15 16:30:41 +0000 2010", "description": "Helping web professionals with #webperf and #webanalytics since 2010.", "favourites_count": 109, "followers_count": 482, "friends_count": 1156, "id": 155955796, "lang": "en", "listed_count": 84, "location": "Gothenburg, Sweden", "name": "Web Analytics Today", "profile_background_color": "000000", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_banner_url": "https://pbs.twimg.com/profile_banners/155955796/1453460635", "profile_image_url": "http://pbs.twimg.com/profile_images/781948051330048002/c7z7T88b_normal.jpg", "profile_link_color": "487680", "profile_sidebar_fill_color": "000000", "profile_text_color": "000000", "screen_name": "AnalyticsCrew", "status": {"created_at": "Fri Apr 28 22:01:19 +0000 2017", "id": 858078701896052736, "id_str": "858078701896052736", "lang": "en", "source": "<a href=\"http://verifierad.nu\" rel=\"nofollow\">Analytics Crew</a>", "text": "Testing 1-2-3!"}, "statuses_count": 1182, "time_zone": "Stockholm", "url": "https://t.co/Zlyc8qZT6x", "utc_offset": 7200}
except:
print('Crap!')
# behöver tidpunkt, hasha tidpunkt och meddelande till logg över vad som gjorts
# leta upp namnet så det går att svara
# ha hashtag för att skilja ut länkar som ska testas från paper.li och annan skit i mentions
# har BeautifulSoup kod för att hitta URLar?
# print(s.text)
try:
print('Tweet: {0}\nPublished: {1}'.format(s.text, s.created_at))
# {"created_at": "Tue Jun 15 16:30:41 +0000 2010", "description": "Helping web professionals with #webperf and #webanalytics since 2010.", "favourites_count": 109, "followers_count": 482, "friends_count": 1156, "id": 155955796, "lang": "en", "listed_count": 84, "location": "Gothenburg, Sweden", "name": "Web Analytics Today", "profile_background_color": "000000", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_banner_url": "https://pbs.twimg.com/profile_banners/155955796/1453460635", "profile_image_url": "http://pbs.twimg.com/profile_images/781948051330048002/c7z7T88b_normal.jpg", "profile_link_color": "487680", "profile_sidebar_fill_color": "000000", "profile_text_color": "000000", "screen_name": "AnalyticsCrew", "status": {"created_at": "Fri Apr 28 22:01:19 +0000 2017", "id": 858078701896052736, "id_str": "858078701896052736", "lang": "en", "source": "<a href=\"http://verifierad.nu\" rel=\"nofollow\">Analytics Crew</a>", "text": "Testing 1-2-3!"}, "statuses_count": 1182, "time_zone": "Stockholm", "url": "https://t.co/Zlyc8qZT6x", "utc_offset": 7200}
except:
print('Crap!')

#status = api.PostUpdate('Testing 1-2-3!')
#print(status.text)
# status = api.PostUpdate('Testing 1-2-3!')
# print(status.text)
10 changes: 5 additions & 5 deletions _privatekeys.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
twitterAccessToken = "CHANGE-ME"
twitterAccessTokenSecret = "CHANGE-ME"

#mySqlServer = ""
#mySqlDbName = ""
#mySqlUid = ""
#mySqlPw = ""
#mySqlTable = ""
# mySqlServer = ""
# mySqlDbName = ""
# mySqlUid = ""
# mySqlPw = ""
# mySqlTable = ""
240 changes: 132 additions & 108 deletions default.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,121 +12,145 @@
import helper

# local variables
#url_for_mainProcess = 'http://vgregion.se/'
i = 1 # global iteration counter

def oneOffProcess(file, test_regime = 'httpStatusCodeCheck'):
""" Inspects a textfile, assuming there's URLs in there, one URL per line.
attributes: file path to open
"""
f = open(file, 'r')

urlsInTextfile = []
iteration_counter = 1
keep_on = True;
time_to_sleep_in_seconds = 90 # TODO: reda ut varför den inte orkar testa flera på raken, begränsning?

output_file = ""

while keep_on:
url = f.readline().replace('\n', '')
mess_to_console = '{0}. {1}'.format(iteration_counter, url)

if len(url) < 7: # break while if line is shorter than seven characters, for instance is http:// or https:// assumed as a prefix
keep_on = False
elif not url.endswith('.pdf'):
# depending on which test regime is chosen
if test_regime == 'httpStatusCodeCheck':
status_code = test.httpStatusCodeCheck(url, False)
print('{0} has a status code: {1}'.format(mess_to_console, status_code).replace('\n', ''))
output_file += '{0}, {1}\n'.format(url.replace('\n', ''), status_code)
elif test_regime == 'mobileFriendlyCheck':
print(url)
status_message = test.mobileFriendlyCheck(url, privatekeys.googleMobileFriendlyApiKey)
print("Mobile-friendliness of URL '{0}' were evaluated as: {1}".format(url, status_message))
output_file += '{0}, {1}\n'.format(url.replace('\n', ''), status_message)
sleep(time_to_sleep_in_seconds) #sleeping for n seconds

urlsInTextfile.append(url)
iteration_counter += 1

f.close()

### Writing the report
file_name = 'rapporter/{0}_{1}_{2}.csv'.format(str(datetime.today())[:10], test_regime, helper.getUniqueId())
helper.writeFile(file_name, output_file)

print('The report has now been written to a file named: {0}'.format(file_name))

def oneOffFromSitemap(url_to_sitemap, check_limit = 50, date_limit = '2017-02-17T06:19:00+01:00', naming = 'google_pagespeed', test_regime = 'googlePageSpeed'):
"""Initially only checks a site against Google Pagespeed API
"""
#urls = set()
urls = helper.fetchUrlsFromSitemap(url_to_sitemap, date_limit)

#print(len(urls))
i = 1
output_file = ''

for url in urls:
mess_to_console = '{0}. {1}'.format(i, url[1])
if i > check_limit:
break
try:
if test_regime == 'googlePageSpeed':
check_page = test.googlePagespeedCheck(url[1])
if bool(check_page):
print('{0} has been checked against Google Pagespeed API'.format(mess_to_console))
for key in check_page:
output_file = output_file + '{0},{1},{2}\n'.format(url[1], key, check_page[key])

i = i + 1
elif test_regime == 'httpStatusCodeCheck':
status_code = test.httpStatusCodeCheck(url[1], False)
print('{0} has a status code: {1}'.format(mess_to_console, status_code))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''), status_code)
i = i + 1
elif test_regime == 'mobileFriendlyCheck':
print(url[1])
status_message = test.mobileFriendlyCheck(url[1], privatekeys.googleMobileFriendlyApiKey)
print("Mobile-friendliness of URL '{0}' were evaluated as: {1}".format(url[1], status_message))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''), status_message)
i = i + 1
elif test_regime == 'thirdPartiesCheck':
print(url[1])
status_message = test.thirdPartiesCheck(url[1])
print("Third parties of URL '{0}' were evaluated as: {1}".format(url[1], status_message))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''), status_message)
i = i + 1
except:
print('Error! The request for URL "{0}" failed.\nMessage:\n{2}'.format(url[1], sys.exc_info()[0]))
pass
i = i + 1

### Writing the report
file_name = 'rapporter/{0}_{1}_{2}.csv'.format(str(datetime.today())[:10], naming, helper.getUniqueId())
helper.writeFile(file_name, output_file)
print('Report written to disk at {0}'.format(file_name))
# url_for_mainProcess = 'http://vgregion.se/'
i = 1 # global iteration counter


def oneOffProcess(file, test_regime='httpStatusCodeCheck'):
""" Inspects a textfile, assuming there's URLs in there, one URL per line.
attributes: file path to open
"""
f = open(file, 'r')

urlsInTextfile = []
iteration_counter = 1
keep_on = True;
time_to_sleep_in_seconds = 90 # TODO: reda ut varför den inte orkar testa flera på raken, begränsning?

output_file = ""

while keep_on:
url = f.readline().replace('\n', '')
mess_to_console = '{0}. {1}'.format(iteration_counter, url)

if len(
url) < 7: # break while if line is shorter than seven characters, for instance is http:// or https:// assumed as a prefix
keep_on = False
elif not url.endswith('.pdf'):
# depending on which test regime is chosen
if test_regime == 'httpStatusCodeCheck':
status_code = test.httpStatusCodeCheck(url, False)
print('{0} has a status code: {1}'.format(mess_to_console,
status_code).replace('\n', ''))
output_file += '{0}, {1}\n'.format(url.replace('\n', ''), status_code)
elif test_regime == 'mobileFriendlyCheck':
print(url)
status_message = test.mobileFriendlyCheck(url,
privatekeys.googleMobileFriendlyApiKey)
print(
"Mobile-friendliness of URL '{0}' were evaluated as: {1}".format(url,
status_message))
output_file += '{0}, {1}\n'.format(url.replace('\n', ''), status_message)
sleep(time_to_sleep_in_seconds) # sleeping for n seconds

urlsInTextfile.append(url)
iteration_counter += 1

f.close()

### Writing the report
file_name = 'rapporter/{0}_{1}_{2}.csv'.format(str(datetime.today())[:10],
test_regime, helper.getUniqueId())
helper.writeFile(file_name, output_file)

print('The report has now been written to a file named: {0}'.format(file_name))


def oneOffFromSitemap(url_to_sitemap, check_limit=50,
date_limit='2017-02-17T06:19:00+01:00', naming='google_pagespeed',
test_regime='googlePageSpeed'):
"""Initially only checks a site against Google Pagespeed API
"""
# urls = set()
urls = helper.fetchUrlsFromSitemap(url_to_sitemap, date_limit)

# print(len(urls))
i = 1
output_file = ''

for url in urls:
mess_to_console = '{0}. {1}'.format(i, url[1])
if i > check_limit:
break
try:
if test_regime == 'googlePageSpeed':
check_page = test.googlePagespeedCheck(url[1])
if bool(check_page):
print('{0} has been checked against Google Pagespeed API'.format(
mess_to_console))
for key in check_page:
output_file = output_file + '{0},{1},{2}\n'.format(url[1], key,
check_page[
key])

i = i + 1
elif test_regime == 'httpStatusCodeCheck':
status_code = test.httpStatusCodeCheck(url[1], False)
print('{0} has a status code: {1}'.format(mess_to_console, status_code))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''), status_code)
i = i + 1
elif test_regime == 'mobileFriendlyCheck':
print(url[1])
status_message = test.mobileFriendlyCheck(url[1],
privatekeys.googleMobileFriendlyApiKey)
print("Mobile-friendliness of URL '{0}' were evaluated as: {1}".format(
url[1], status_message))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''),
status_message)
i = i + 1
elif test_regime == 'thirdPartiesCheck':
print(url[1])
status_message = test.thirdPartiesCheck(url[1])
print("Third parties of URL '{0}' were evaluated as: {1}".format(url[1],
status_message))
output_file += '{0}, {1}\n'.format(url[1].replace('\n', ''),
status_message)
i = i + 1
except:
print('Error! The request for URL "{0}" failed.\nMessage:\n{2}'.format(url[1],
sys.exc_info()[
0]))
pass
i = i + 1

### Writing the report
file_name = 'rapporter/{0}_{1}_{2}.csv'.format(str(datetime.today())[:10], naming,
helper.getUniqueId())
helper.writeFile(file_name, output_file)
print('Report written to disk at {0}'.format(file_name))


# supposed to support scheduling from bash-scripts or hosts such as PythonAnywhere
def checkSitemapsForNewUrls(file):
""" Checking a list of predefined sitemaps for new or updated URLs
Attributes: string file (for the file location on disk)
"""
#iterera runt de URLar som finns och anropa sitemaps
#kolla om det finns material som är mindre än 14 dagar gammalt (i slutändan kör man denna dagligen per sajt, typ)
""" Checking a list of predefined sitemaps for new or updated URLs
Attributes: string file (for the file location on disk)
"""


# bygga ett register över URLars ålder
# iterera runt de URLar som finns och anropa sitemaps
# kolla om det finns material som är mindre än 14 dagar gammalt (i slutändan kör man denna dagligen per sajt, typ)

# om ny URL hittas så läggs den i en textfil som kollas på i slutet av exekveringen
# bygga ett register över URLars ålder

# om ny URL hittas så läggs den i en textfil som kollas på i slutet av exekveringen

"""
If file is executed on itself then call on a definition
"""
if __name__ == '__main__':
#oneOffProcess('exempelfiler/swe-gov.txt', 'httpStatusCodeCheck')
#oneOffFromSitemap('http://www.vgregion.se/sitemap.xml', 100, 'vgregion-httpStatusCodeCheck', 'httpStatusCodeCheck')
oneOffFromSitemap('http://www.varberg.se/sitemap.xml', 10, '2017-02-17T06:19:00+01:00', 'pagespeed', 'googlePageSpeed')
# oneOffProcess('exempelfiler/swe-gov.txt', 'httpStatusCodeCheck')
# oneOffFromSitemap('http://www.vgregion.se/sitemap.xml', 100, 'vgregion-httpStatusCodeCheck', 'httpStatusCodeCheck')
oneOffFromSitemap('http://www.varberg.se/sitemap.xml', 10,
'2017-02-17T06:19:00+01:00', 'pagespeed', 'googlePageSpeed')
Loading

0 comments on commit e7e1f1a

Please sign in to comment.