Skip to content

Commit

Permalink
Merge 24ed123 into adb85a7
Browse files Browse the repository at this point in the history
  • Loading branch information
@lrromero
lrromero committed Dec 9, 2019
2 parents adb85a7 + 24ed123 commit 1a1090e
Show file tree
Hide file tree
Showing 8 changed files with 17,351 additions and 1,029 deletions.
48 changes: 21 additions & 27 deletions pydatajson/federation.py
View file
Expand Up @@ -13,7 +13,8 @@

from pydatajson.constants import REQUESTS_TIMEOUT, DEFAULT_TIMEZONE
from pydatajson.custom_exceptions import NumericDistributionIdentifierError
from .ckan_utils import map_dataset_to_package, map_theme_to_group
from .ckan_utils import map_dataset_to_package, map_theme_to_group,\
map_distributions_to_resources
from pydatajson.custom_remote_ckan import CustomRemoteCKAN as RemoteCKAN
from .search import get_datasets
from .helpers import resource_files_download
Expand Down Expand Up @@ -136,30 +137,23 @@ def resources_update(portal_url, apikey, distributions,
requests_timeout=requests_timeout)
result = []
generate_new_access_url = generate_new_access_url or []
for distribution in distributions:
updated = False
resource_id = catalog_id + '_' + distribution['identifier']\
if catalog_id else distribution['identifier']
fields = {'id': resource_id}
if distribution['identifier'] in generate_new_access_url:
fields.update({'accessURL': ''})
updated = True
if distribution['identifier'] in resource_files:
fields.update({'resource_type': 'file.upload',
'upload':
open(resource_files[distribution['identifier']],
'rb')
})
updated = True
if updated:
try:
pushed = ckan_portal.action.resource_patch(**fields)
result.append(pushed['id'])
except CKANAPIError as e:
logger.exception(
"Error subiendo recurso {} a la distribución {}: {}"
.format(resource_files[distribution['identifier']],
resource_files, str(e)))
ckan_resources = map_distributions_to_resources(distributions)
for resource in ckan_resources:
if resource['id'] in generate_new_access_url:
resource.update({'accessURL': ''})
if resource['id'] in resource_files:
resource.update({'resource_type': 'file.upload',
'upload':
open(resource_files[resource['id']], 'rb')})
resource['id'] = catalog_id + '_' + resource['id'] \
if catalog_id else resource['id']
try:
pushed = ckan_portal.action.resource_patch(**resource)
result.append(pushed['id'])
except CKANAPIError as e:
logger.exception(
"Error actualizando distribución {}: {}"
.format(resource['id'], str(e)))
return result


Expand Down Expand Up @@ -439,8 +433,8 @@ def push_new_themes(catalog, portal_url, apikey):
verify_ssl=catalog.verify_ssl,
requests_timeout=catalog.requests_timeout)
existing_themes = ckan_portal.call_action('group_list')
new_themes = [theme['id'] for theme in catalog[
'themeTaxonomy'] if theme['id'] not in existing_themes]
new_themes = [theme['id'] for theme in catalog.get('themeTaxonomy', [])
if theme['id'] not in existing_themes]
pushed_names = []
for new_theme in new_themes:
name = push_theme_to_ckan(
Expand Down
191 changes: 191 additions & 0 deletions tests/cassetes/backup/test_make_catalog_backup_data.yaml
View file
Expand Up @@ -199,4 +199,195 @@ interactions:
x-timer: ['S1523912092.872041,VS0,VE0']
x-xss-protection: [1; mode=block]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python-requests/2.22.0]
method: GET
uri: https://github.com/datosgobar/pydatajson/raw/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv
response:
body: {string: !!python/unicode '<html><body>You are being <a href="https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv">redirected</a>.</body></html>'}
headers:
access-control-allow-origin: ['https://render.githubusercontent.com']
age: ['0']
cache-control: [no-cache]
content-security-policy: ['default-src ''none''; base-uri ''self''; block-all-mixed-content;
connect-src ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com
api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com
github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com
wss://live.github.com; font-src github.githubassets.com; form-action ''self''
github.com gist.github.com; frame-ancestors ''none''; frame-src render.githubusercontent.com;
img-src ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com
github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self'';
media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline''
github.githubassets.com']
content-type: [text/html; charset=utf-8]
date: ['Mon, 09 Dec 2019 13:06:11 GMT']
expect-ct: ['max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"']
location: ['https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv']
server: [GitHub.com]
status: [302 Found]
strict-transport-security: [max-age=31536000; includeSubdomains; preload]
vary: [X-PJAX]
x-content-type-options: [nosniff]
x-frame-options: [deny]
x-github-request-id: ['8DDA:4785:4E254AD:83C0732:5DEE46C2']
x-xss-protection: [1; mode=block]
status: {code: 302, message: Found}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python-requests/2.22.0]
method: GET
uri: https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv
response:
body:
string: !!binary |
H4sIAAAAAAAAA1VW2W4jOQx8n2/pBSQeIvU1QQ4PYGBjB4lnMZ8/VTo6s0HQcMoqkUUW2bne3q6v
l6fH9fL+cT/uPy+fj+enx/3x/O9TK8fH9WX98fHOv684hAOv1/vt8kXg7fL+fHv7i/F6v339er+P
w7f/Lp9fOPr08vkLt15vj8vn7ZlfXX7//54ftUf5p1T8HtVD+lFN3Y6q4Rso5n6oazlqmC2GDUaI
Fx4zxbPEBqrhqa3ZIVXbYsRgtN5xpVs2PKtugFEPDQOaNSejlhmjWOIsfpBb1w2QeGh25Ir8yKhb
R1PDEyH4XeQGalVFjMbvLBdj6sjqiNzMx/W6ASlmh6ngnshYjKkjxSC0OTNuTGIC4ohktQky1hVj
6YAuoC0byqTVNyA4fZgINWonQ7aOKOyENypXVmkCtSOGMhAbtBhLh2pnDJbJRjEHIJBzaE8yTBdj
6WiCJKLyYmu5AS3FmBXIqPFkbB1dk07osmMMQDL6YU2JyqiVnjpyZOU8pt+AqKGD0YMV1MWYOnqp
zCo6L662AXUkZKUk+9oWY+lI5cWtoFYisQE1gROTjFZsMpaO3pNyrQbreAJWEjE0Wfk6YtjWkTaU
CydBSm5ANQNun13ajKED/uGxXshQOHEB1hpd4ux55mLEZCiFZtKEXssGYGOH8t44nzIZU4dAGM52
Rx2llLYB6xaHjXvSh3LfOnprPhwd47kBS4jWYLlLi8WYOmopSKILl0KgugtwF3gXHWJrfDFmP/qM
LHS7a9sADIMZRDzWY2V19oNjnU7vKqdvAlYqauVjJ+VgtO9+IDzyYa8qV9AE1KAG/aB3iyyGbeWs
LhYWNLLbE8AwobrwCj3ni7F0BHuX2kYHywYUpUZWOtOcjK0jmGs67SeiG5DEtI/Gi9SRVZw6kj4P
FzoodANS0H/jkMjaDHH2o/ahZuxdHpiABMYPPWmH1jlRsX1VnRum98KthY8LgLnycKdtcuqI7auS
Nns+dlDfgDboH7uV80tGnjrQC26fHOvANoDUfO5dsTkfec5HYUmT5gbDNyAFHrPoGF1IX4w9H6xS
du5k+wawPxP9gKPR/ToZW4d1HOjTpRCzAMUyQYyAcm1jBvvSIcVZpUxONN4JG1CBVxADQ9lOxtSB
HSW8mPsy8FabAKoLTS5YkhiruhhTh5iM5Dt5oE0AktBNLxxEyRVj6RC+hnEWxRS+aybAiQLDiCK1
H5yzrQO7nPYzHjY4cQHKtepYKYf08a7Ncuoozp5zdYrICajjeuxIpJknY+nA57HW8HR+HAAux/vK
uS+kF5+MrYNrBo3GnEv0DeDfB2wUZAVG4M35BzheK/gfCQAA
headers:
accept-ranges: [bytes]
access-control-allow-origin: ['*']
cache-control: [max-age=300]
connection: [keep-alive]
content-encoding: [gzip]
content-length: ['1005']
content-security-policy: [default-src 'none'; style-src 'unsafe-inline'; sandbox]
content-type: [text/plain; charset=utf-8]
date: ['Mon, 09 Dec 2019 13:06:11 GMT']
etag: [W/"668ff3c567ec8ef5b0fb38703685fe95e1ee8372125e8ab4486c45c106548d9a"]
expires: ['Mon, 09 Dec 2019 13:11:11 GMT']
source-age: ['0']
strict-transport-security: [max-age=31536000]
vary: ['Authorization,Accept-Encoding']
via: [1.1 varnish]
x-cache: [MISS]
x-cache-hits: ['0']
x-content-type-options: [nosniff]
x-fastly-request-id: [76d32c2ca2fd97b90c6a0c10239a34c15373f669]
x-frame-options: [deny]
x-geo-block-list: ['']
x-github-request-id: ['CA68:0537:AC8F9E:D1549F:5DEE46C1']
x-served-by: [cache-eze19324-EZE]
x-timer: ['S1575896772.599100,VS0,VE240']
x-xss-protection: [1; mode=block]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python-requests/2.22.0]
method: GET
uri: https://github.com/datosgobar/pydatajson/raw/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv
response:
body: {string: !!python/unicode '<html><body>You are being <a href="https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv">redirected</a>.</body></html>'}
headers:
access-control-allow-origin: ['https://render.githubusercontent.com']
age: ['0']
cache-control: [no-cache]
content-length: ['195']
content-security-policy: ['default-src ''none''; base-uri ''self''; block-all-mixed-content;
connect-src ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com
api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com
github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com
wss://live.github.com; font-src github.githubassets.com; form-action ''self''
github.com gist.github.com; frame-ancestors ''none''; frame-src render.githubusercontent.com;
img-src ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com
github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self'';
media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline''
github.githubassets.com']
content-type: [text/html; charset=utf-8]
date: ['Mon, 09 Dec 2019 13:06:12 GMT']
expect-ct: ['max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"']
location: ['https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv']
server: [GitHub.com]
status: [302 Found]
strict-transport-security: [max-age=31536000; includeSubdomains; preload]
vary: [X-PJAX]
x-content-type-options: [nosniff]
x-frame-options: [deny]
x-github-request-id: ['8DDE:591E:3055C14:55A5125:5DEE46C4']
x-xss-protection: [1; mode=block]
status: {code: 302, message: Found}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python-requests/2.22.0]
method: GET
uri: https://raw.githubusercontent.com/datosgobar/pydatajson/master/samples/series-tiempo/odg-total-millones-pesos-1960-trimestral.csv
response:
body:
string: !!binary |
H4sIAAAAAAAAA1VW2W4jOQx8n2/pBSQeIvU1QQ4PYGBjB4lnMZ8/VTo6s0HQcMoqkUUW2bne3q6v
l6fH9fL+cT/uPy+fj+enx/3x/O9TK8fH9WX98fHOv684hAOv1/vt8kXg7fL+fHv7i/F6v339er+P
w7f/Lp9fOPr08vkLt15vj8vn7ZlfXX7//54ftUf5p1T8HtVD+lFN3Y6q4Rso5n6oazlqmC2GDUaI
Fx4zxbPEBqrhqa3ZIVXbYsRgtN5xpVs2PKtugFEPDQOaNSejlhmjWOIsfpBb1w2QeGh25Ir8yKhb
R1PDEyH4XeQGalVFjMbvLBdj6sjqiNzMx/W6ASlmh6ngnshYjKkjxSC0OTNuTGIC4ohktQky1hVj
6YAuoC0byqTVNyA4fZgINWonQ7aOKOyENypXVmkCtSOGMhAbtBhLh2pnDJbJRjEHIJBzaE8yTBdj
6WiCJKLyYmu5AS3FmBXIqPFkbB1dk07osmMMQDL6YU2JyqiVnjpyZOU8pt+AqKGD0YMV1MWYOnqp
zCo6L662AXUkZKUk+9oWY+lI5cWtoFYisQE1gROTjFZsMpaO3pNyrQbreAJWEjE0Wfk6YtjWkTaU
CydBSm5ANQNun13ajKED/uGxXshQOHEB1hpd4ux55mLEZCiFZtKEXssGYGOH8t44nzIZU4dAGM52
Rx2llLYB6xaHjXvSh3LfOnprPhwd47kBS4jWYLlLi8WYOmopSKILl0KgugtwF3gXHWJrfDFmP/qM
LHS7a9sADIMZRDzWY2V19oNjnU7vKqdvAlYqauVjJ+VgtO9+IDzyYa8qV9AE1KAG/aB3iyyGbeWs
LhYWNLLbE8AwobrwCj3ni7F0BHuX2kYHywYUpUZWOtOcjK0jmGs67SeiG5DEtI/Gi9SRVZw6kj4P
FzoodANS0H/jkMjaDHH2o/ahZuxdHpiABMYPPWmH1jlRsX1VnRum98KthY8LgLnycKdtcuqI7auS
Nns+dlDfgDboH7uV80tGnjrQC26fHOvANoDUfO5dsTkfec5HYUmT5gbDNyAFHrPoGF1IX4w9H6xS
du5k+wawPxP9gKPR/ToZW4d1HOjTpRCzAMUyQYyAcm1jBvvSIcVZpUxONN4JG1CBVxADQ9lOxtSB
HSW8mPsy8FabAKoLTS5YkhiruhhTh5iM5Dt5oE0AktBNLxxEyRVj6RC+hnEWxRS+aybAiQLDiCK1
H5yzrQO7nPYzHjY4cQHKtepYKYf08a7Ncuoozp5zdYrICajjeuxIpJknY+nA57HW8HR+HAAux/vK
uS+kF5+MrYNrBo3GnEv0DeDfB2wUZAVG4M35BzheK/gfCQAA
headers:
accept-ranges: [bytes]
access-control-allow-origin: ['*']
cache-control: [max-age=300]
connection: [keep-alive]
content-encoding: [gzip]
content-length: ['1005']
content-security-policy: [default-src 'none'; style-src 'unsafe-inline'; sandbox]
content-type: [text/plain; charset=utf-8]
date: ['Mon, 09 Dec 2019 13:06:13 GMT']
etag: [W/"668ff3c567ec8ef5b0fb38703685fe95e1ee8372125e8ab4486c45c106548d9a"]
expires: ['Mon, 09 Dec 2019 13:11:13 GMT']
source-age: ['1']
strict-transport-security: [max-age=31536000]
vary: ['Authorization,Accept-Encoding']
via: [1.1 varnish]
x-cache: [HIT]
x-cache-hits: ['1']
x-content-type-options: [nosniff]
x-fastly-request-id: [3b4aa731eee932ed62608963dd9c74463e6b7dcc]
x-frame-options: [deny]
x-geo-block-list: ['']
x-github-request-id: ['CA68:0537:AC8F9E:D1549F:5DEE46C1']
x-served-by: [cache-eze19324-EZE]
x-timer: ['S1575896773.009472,VS0,VE0']
x-xss-protection: [1; mode=block]
status: {code: 200, message: OK}
version: 1

0 comments on commit 1a1090e

Please sign in to comment.