Skip to content

Commit

Permalink
Agrego parametros de timeout y verficación de ssl
Browse files Browse the repository at this point in the history
  • Loading branch information
@lrromero
lrromero committed May 30, 2019
1 parent d9f7880 commit 5d8eaa6
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 19 deletions.
21 changes: 21 additions & 0 deletions pydatajson/ckan_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
import re
import logging
from datetime import time

from ckanapi import RemoteCKAN
from dateutil import parser, tz
from .helpers import title_to_name
from . import custom_exceptions as ce
from .constants import REQUESTS_TIMEOUT

logger = logging.getLogger('pydatajson')

Expand Down Expand Up @@ -167,3 +170,21 @@ def map_theme_to_group(theme):
"title": theme.get('label'),
"description": theme.get('description'),
}


class CustomRemoteCKAN(RemoteCKAN):

def __init__(self, address, apikey=None, user_agent=None, get_only=False,
verify_ssl=False, requests_timeout=REQUESTS_TIMEOUT):
self.verify_ssl = verify_ssl
self.requests_timeout = requests_timeout
super(CustomRemoteCKAN, self).__init__(address, apikey,
user_agent, get_only)

def call_action(self, action, data_dict=None, context=None, apikey=None,
files=None, requests_kwargs=None):
requests_kwargs = requests_kwargs or {}
requests_kwargs.setdefault('verify', self.verify_ssl)
requests_kwargs.setdefault('timeout', self.requests_timeout)
return super(CustomRemoteCKAN, self).call_action(
action, data_dict, context, apikey, files, requests_kwargs)
82 changes: 63 additions & 19 deletions pydatajson/federation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@

from __future__ import print_function, unicode_literals
import logging
from ckanapi import RemoteCKAN
from ckanapi.errors import NotFound, CKANAPIError

from pydatajson.constants import REQUESTS_TIMEOUT
from .ckan_utils import map_dataset_to_package, map_theme_to_group
from .ckan_utils import CustomRemoteCKAN as RemoteCKAN
from .search import get_datasets
from .helpers import resource_files_download

Expand Down Expand Up @@ -47,7 +49,9 @@ def push_dataset_to_ckan(catalog, owner_org, dataset_origin_identifier,
str: El id del dataset en el catálogo de destino.
"""
dataset = catalog.get_dataset(dataset_origin_identifier)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey,
verify_ssl=catalog.verify_ssl,
requests_timeout=catalog.requests_timeout)

package = map_dataset_to_package(catalog, dataset, owner_org, catalog_id,
demote_superThemes, demote_themes)
Expand Down Expand Up @@ -85,7 +89,8 @@ def push_dataset_to_ckan(catalog, owner_org, dataset_origin_identifier,

def resources_update(portal_url, apikey, distributions,
resource_files, generate_new_access_url=None,
catalog_id=None):
catalog_id=None, verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Sube archivos locales a sus distribuciones correspondientes en el portal
pasado por parámetro.
Expand All @@ -102,10 +107,14 @@ def resources_update(portal_url, apikey, distributions,
generados por el portal de destino
catalog_id(str): prependea el id al id del recurso para
encontrarlo antes de subirlo
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
list: los ids de los recursos modificados
"""
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
result = []
generate_new_access_url = generate_new_access_url or []
for distribution in distributions:
Expand Down Expand Up @@ -135,8 +144,10 @@ def resources_update(portal_url, apikey, distributions,
return result


def remove_dataset_from_ckan(identifier, portal_url, apikey):
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
def remove_dataset_from_ckan(identifier, portal_url, apikey, verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
ckan_portal = RemoteCKAN(portal_url, apikey=apikey, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
ckan_portal.call_action('dataset_purge', data_dict={'id': identifier})


Expand All @@ -162,7 +173,8 @@ def remove_harvested_ds_from_ckan(catalog, portal_url, apikey,

def remove_datasets_from_ckan(portal_url, apikey, filter_in=None,
filter_out=None, only_time_series=False,
organization=None):
organization=None, verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Borra un dataset en el portal pasado por parámetro.
Args:
Expand All @@ -177,10 +189,14 @@ def remove_datasets_from_ckan(portal_url, apikey, filter_in=None,
recursos con series de tiempo.
organization(str): Filtrar solo los datasets que pertenezcan a
cierta organizacion.
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
None
"""
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
identifiers = []
datajson_filters = filter_in or filter_out or only_time_series
if datajson_filters:
Expand Down Expand Up @@ -229,7 +245,9 @@ def push_theme_to_ckan(catalog, portal_url, apikey,
Returns:
str: El name del theme en el catálogo de destino.
"""
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey,
verify_ssl=catalog.verify_ssl,
requests_timeout=catalog.requests_timeout)
theme = catalog.get_theme(identifier=identifier, label=label)
group = map_theme_to_group(theme)
pushed_group = ckan_portal.call_action('group_create', data_dict=group)
Expand Down Expand Up @@ -357,7 +375,9 @@ def push_new_themes(catalog, portal_url, apikey):
Returns:
str: Los ids de los temas creados.
"""
ckan_portal = RemoteCKAN(portal_url, apikey=apikey)
ckan_portal = RemoteCKAN(portal_url, apikey=apikey,
verify_ssl=catalog.verify_ssl,
requests_timeout=catalog.requests_timeout)
existing_themes = ckan_portal.call_action('group_list')
new_themes = [theme['id'] for theme in catalog[
'themeTaxonomy'] if theme['id'] not in existing_themes]
Expand All @@ -369,30 +389,40 @@ def push_new_themes(catalog, portal_url, apikey):
return pushed_names


def get_organizations_from_ckan(portal_url):
def get_organizations_from_ckan(portal_url, verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Toma la url de un portal y devuelve su árbol de organizaciones.
Args:
portal_url (str): La URL del portal CKAN de origen.
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
list: Lista de diccionarios anidados con la información de
las organizaciones.
"""
ckan_portal = RemoteCKAN(portal_url)
ckan_portal = RemoteCKAN(portal_url, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
return ckan_portal.call_action('group_tree',
data_dict={'type': 'organization'})


def get_organization_from_ckan(portal_url, org_id):
def get_organization_from_ckan(portal_url, org_id, verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Toma la url de un portal y un id, y devuelve la organización a buscar.
Args:
portal_url (str): La URL del portal CKAN de origen.
org_id (str): El id de la organización a buscar.
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
dict: Diccionario con la información de la organización.
"""
ckan_portal = RemoteCKAN(portal_url)
ckan_portal = RemoteCKAN(portal_url, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
return ckan_portal.call_action('organization_show',
data_dict={'id': org_id})

Expand Down Expand Up @@ -428,21 +458,27 @@ def push_organization_tree_to_ckan(portal_url, apikey, org_tree, parent=None):
return created


def push_organization_to_ckan(portal_url, apikey, organization, parent=None):
def push_organization_to_ckan(portal_url, apikey, organization, parent=None,
verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Toma una organización y la crea en el portal de destino.
Args:
portal_url (str): La URL del portal CKAN de destino.
apikey (str): La apikey de un usuario con los permisos que le
permitan crear la organización.
organization(dict): Datos de la organización a crear.
parent(str): Campo name de la organización padre.
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
(dict): Devuelve el diccionario de la organizacion enviada,
junto con el status detallando si la creación fue
exitosa o no.
"""
portal = RemoteCKAN(portal_url, apikey=apikey)
portal = RemoteCKAN(portal_url, apikey=apikey, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
if parent:
organization['groups'] = [{'name': parent}]
try:
Expand All @@ -457,18 +493,24 @@ def push_organization_to_ckan(portal_url, apikey, organization, parent=None):
return pushed_org


def remove_organization_from_ckan(portal_url, apikey, organization_id):
def remove_organization_from_ckan(portal_url, apikey, organization_id,
verify_ssl=False,
requests_timeout=REQUESTS_TIMEOUT):
"""Toma un id de organización y la purga del portal de destino.
Args:
portal_url (str): La URL del portal CKAN de destino.
apikey (str): La apikey de un usuario con los permisos que le
permitan borrar la organización.
organization_id(str): Id o name de la organización a borrar.
verify_ssl(bool): Verificar certificados SSL
requests_timeout(int): cantidad en segundos para timeoutear un
request al server.
Returns:
None.
"""
portal = RemoteCKAN(portal_url, apikey=apikey)
portal = RemoteCKAN(portal_url, apikey=apikey, verify_ssl=verify_ssl,
requests_timeout=requests_timeout)
try:
portal.call_action('organization_purge',
data_dict={'id': organization_id})
Expand Down Expand Up @@ -570,7 +612,9 @@ def restore_catalog_to_ckan(catalog, origin_portal_url, destination_portal_url,
"""
catalog['homepage'] = catalog.get('homepage') or origin_portal_url
res = {}
origin_portal = RemoteCKAN(origin_portal_url)
origin_portal = RemoteCKAN(origin_portal_url,
verify_ssl=catalog.verify_ssl,
requests_timeout=catalog.requests_timeout)

try:
org_list = origin_portal.action.organization_list()
Expand Down

0 comments on commit 5d8eaa6

Please sign in to comment.