Authenticate requests for files uploaded to CKAN

If the resource url_type is uploaded, we send the Authenticatio header
with the request, as we are actually pointing to a CKAN controller. If
we don't provide it, the request will fail as CKAN will return a 401

datapusher/jobs.py

@@ -234,8 +234,14 @@ def push_to_datastore(task_id, input, dry_run=False):
     # fetch the resource data
     logger.info('Fetching from: {0}'.format(resource.get('url')))
     try:
-        response = urllib2.urlopen(resource.get('url'),
-                                   timeout=DOWNLOAD_TIMEOUT)
+        request = urllib2.Request(resource.get('url'))
+
+        if resource.get('url_type') == 'upload':
+            # If this is an uploaded file to CKAN, authenticate the request,
+            # otherwise we won't get file from private resources
+            request.add_header('Authorization', api_key)
+
+        response = urllib2.urlopen(request, timeout=DOWNLOAD_TIMEOUT)
     except urllib2.HTTPError as e:
         raise util.JobError('Invalid HTTP response: %s' % e)
     except urllib2.URLError as e: