diff --git a/pipeline.py b/pipeline.py index 28d8f0a..139bf00 100644 --- a/pipeline.py +++ b/pipeline.py @@ -59,7 +59,7 @@ # Update this each time you make a non-cosmetic change. # It will be added to the WARC files and reported to the tracker. -VERSION = '20181216.04' +VERSION = '20181216.05' USER_AGENT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html; ArchiveTeam)' TRACKER_ID = 'tumblr' TRACKER_HOST = 'tracker.archiveteam.org' diff --git a/tumblr.lua b/tumblr.lua index 90a90ca..e2aded9 100644 --- a/tumblr.lua +++ b/tumblr.lua @@ -284,7 +284,11 @@ wget.callbacks.httploop_result = function(url, err, http_stat) end end - if status_code == 403 or status_code == 400 or status_code == 0 then + if status_code == 403 -- banned + or status_code == 400 -- ? + or status_code == 429 -- rate limit exceeded + or status_code == 0 -- download error + then --if string.match(url["host"], "") if string.match(url["host"], "assets%.tumblr%.com") or string.match(url["host"], "static%.tumblr%.com")