Skip to content

Commit

Permalink
improve downlod of hosted link
Browse files Browse the repository at this point in the history
  • Loading branch information
@Guts
Guts committed Apr 3, 2018
1 parent d9774b9 commit 0c4c5fb
Show file tree
Hide file tree
Showing 6 changed files with 200 additions and 69 deletions.
14 changes: 14 additions & 0 deletions isogeo_pysdk/checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,20 @@ def _check_filter_specific_md(self, specific_md):
raise TypeError("'specific_md' expects a list")
return specific_md

def _check_filter_specific_tag(self, specific_tag):
"""Check if specific_tag parameter is valid.
:param list specific_tag: list of specific tag to check
"""
if isinstance(specific_tag, list):
if len(specific_tag) > 0:
specific_tag = ",".join(specific_tag)
else:
specific_tag = ""
else:
raise TypeError("'specific_tag' expects a list")
return specific_tag

def _check_filter_sub_resources(self, sub_resources, resource="metadata"):
"""Check if specific_resources parameter is valid.
Expand Down
46 changes: 22 additions & 24 deletions isogeo_pysdk/isogeo_sdk.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,9 @@ def licenses(self, token, owner_id, prot="https"):
(use it only for dev and tracking needs).
"""
# checking bearer validity
token = checker.check_bearer_validity(token, self.connect(self.app_id, self.ct))
token = checker.check_bearer_validity(token,
self.connect(self.app_id,
self.ct))

# handling request parameters
payload = {'gid': owner_id,
Expand Down Expand Up @@ -632,19 +634,11 @@ def keywords(self,

# specific resources specific parsing
specific_md = checker._check_filter_specific_md(specific_md)

# specific tag specific parsing
if isinstance(specific_tag, list):
if len(specific_tag) > 0:
specific_tag = ",".join(specific_tag)
else:
specific_tag = ""
else:
raise TypeError("'specific_tag' expects a list")

# sub resources specific parsing
sub_resources = checker._check_filter_sub_resources(sub_resources,
"keyword")
# specific tag specific parsing
specific_tag = checker._check_filter_specific_tag(specific_tag)

# handling request parameters
payload = {'_id': specific_md,
Expand Down Expand Up @@ -680,23 +674,17 @@ def keywords(self,

# -- DOWNLOADS -----------------------------------------------------------

def dl_hosted(self, token, id_resource, resource_link,
def dl_hosted(self, token, resource_link,
proxy_url=None, prot="https"):
"""Download hosted resource.
:param str token: API auth token
:param str id_resource: metadata UUID
:param dict resource_link: link dictionary
:param str token: API auth token
:param str proxy_url: proxy to use to download
:param str prot: https [DEFAULT] or http
(use it only for dev and tracking needs).
"""
# check metadata UUID
if not checker.check_is_uuid(id_resource):
raise ValueError("Metadata ID is not a correct UUID.")
else:
pass
# check resource link parameter type
if not isinstance(resource_link, dict):
raise TypeError("Resource link expects a dictionary.")
Expand All @@ -707,7 +695,7 @@ def dl_hosted(self, token, id_resource, resource_link,
raise ValueError("Resource link passed is not a hosted one: {}"
.format(resource_link.get("type")))
else:
id_link = resource_link.get("_id")
pass

# checking bearer validity
token = checker.check_bearer_validity(token,
Expand All @@ -720,22 +708,32 @@ def dl_hosted(self, token, id_resource, resource_link,
# prepare URL request
head = {"Authorization": "Bearer " + token[0],
"user-agent": self.app_name}
hosted_url = "{}://v1.{}.isogeo.com/resources/{}/links/{}.bin"\

hosted_url = "{}://v1.{}.isogeo.com/{}"\
.format(prot,
self.base_url,
id_resource,
id_link)
resource_link.get("url"))

# send stream request
hosted_req = requests.get(hosted_url,
headers=head,
stream=True,
params=payload,
proxies=self.proxies,
verify=self.ssl)
# quick check
req_check = checker.check_api_response(hosted_req)
if not req_check:
raise requests.exceptions.ConnectionError(req_check[1])
else:
pass

# get filename from header
content_disposition = hosted_req.headers.get("Content-Disposition")
filename = re.findall("filename=(.+)", content_disposition)
if content_disposition:
filename = re.findall("filename=(.+)", content_disposition)[0]
else:
filename = resource_link.get("title")

# well-formed size
in_size = resource_link.get("size")
Expand All @@ -747,7 +745,7 @@ def dl_hosted(self, token, id_resource, resource_link,
out_size = "%3.1f %s" % (in_size, " To")

# end of method
return (hosted_req, filename[0], out_size)
return (hosted_req, filename, out_size)

def xml19139(self, token, id_resource, proxy_url=None, prot="https"):
"""Get resource exported into XML ISO 19139.
Expand Down
20 changes: 17 additions & 3 deletions isogeo_pysdk/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,18 @@ class IsogeoUtils(object):
"qa": "api.qa",
}

APP_URLS = {"prod": "https://app.isogeo.com",
"qa": "https://qa-isogeo-app.azurewebsites.net",
}

MNG_URLS = {"prod": "https://manage.isogeo.com",
"qa": "https://qa-isogeo-manage.azurewebsites.net",
}

OC_URLS = {"prod": "https://open.isogeo.com",
"qa": "https://qa-isogeo-open.azurewebsites.net",
}

WEBAPPS = {"oc": {"args": ("md_id", "share_id", "oc_token"),
"url": "https://open.isogeo.com/s/{share_id}"
"/{oc_token}/r/{md_id}"
Expand Down Expand Up @@ -190,10 +202,12 @@ def get_edit_url(self, md_id=str, md_type=str, owner_id=str, tab="identification
if checker.check_edit_tab(tab, md_type=md_type):
pass
# construct URL
return "https://app.isogeo.com/" \
"groups/{}" \
return "{}" \
"/groups/{}" \
"/resources/{}" \
"/{}".format(owner_id, md_id, tab)
"/{}".format(self.APP_URLS.get(self.platform),
owner_id,
md_id, tab)

def get_view_url(self, webapp="oc", **kwargs):
"""Constructs the view URL of a metadata.
Expand Down
64 changes: 62 additions & 2 deletions tests/test_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,9 +266,22 @@ def test_check_edit_tab_bad(self):
with self.assertRaises(ValueError):
checker.check_edit_tab(tab="attributes", md_type="service")

# requests parameters
def test_check_filter_sub_resources_ok(self):
"""Return"""
# metadata sub resources
"""Check sub resources"""
# metadata sub resources - empty
subresources = checker._check_filter_sub_resources(sub_resources=[],
resource="metadata")
self.assertIsInstance(subresources, string_types)
# metadata sub resources - 1
subresources = checker._check_filter_sub_resources(sub_resources=["links", ],
resource="metadata")
self.assertIsInstance(subresources, string_types)
# metadata sub resources - >1
subresources = checker._check_filter_sub_resources(sub_resources=["contacts", "links"],
resource="metadata")
self.assertIsInstance(subresources, string_types)
# metadata sub resources - all
subresources = checker._check_filter_sub_resources(sub_resources="all",
resource="metadata")
self.assertIsInstance(subresources, string_types)
Expand All @@ -282,6 +295,53 @@ def test_check_filter_sub_resources_bad(self):
with self.assertRaises(ValueError):
checker._check_filter_sub_resources(sub_resources="all",
resource="Metadata")
with self.assertRaises(TypeError):
checker._check_filter_sub_resources(sub_resources="layers",
resource="metadata")

def test_check_filter_specific_md_ok(self):
"""Check specific md"""
uuid_sample_1 = "0269803d50c446b09f5060ef7fe3e22b"
uuid_sample_2 = "0269803d50c446b09f5060ef7fe3e22a"
# metadata sub resources - empty
check = checker._check_filter_specific_md(specific_md=[])
self.assertIsInstance(check, string_types)
# metadata sub resources - 1
check = checker._check_filter_specific_md(specific_md=[uuid_sample_1, ])
self.assertIsInstance(check, string_types)
# metadata sub resources - >1
check = checker._check_filter_specific_md(specific_md=[uuid_sample_1, uuid_sample_2])
self.assertIsInstance(check, string_types)
# metadata sub resources - with bad uuid
check = checker._check_filter_specific_md(specific_md=[uuid_sample_1, "uuid_sample_2"])
self.assertIsInstance(check, string_types)

def test_check_filter_specific_md_bad(self):
"""Raise errors"""
with self.assertRaises(TypeError):
checker._check_filter_specific_md(specific_md="oh_yeah_i_m_a_metadata_uuid")

def test_check_filter_specific_tag_ok(self):
"""Check specific tag"""
kw_sample_1 = "keyword:isogeo:demographie"
kw_sample_2 = "keyword:isogeo:2014"
# metadata sub resources - empty
check = checker._check_filter_specific_tag(specific_tag=[])
self.assertIsInstance(check, string_types)
# metadata sub resources - 1
check = checker._check_filter_specific_tag(specific_tag=[kw_sample_1, ])
self.assertIsInstance(check, string_types)
# metadata sub resources - >1
check = checker._check_filter_specific_tag(specific_tag=[kw_sample_1, kw_sample_2])
self.assertIsInstance(check, string_types)
# metadata sub resources - with bad uuid
check = checker._check_filter_specific_tag(specific_tag=[kw_sample_1, "kw_sample_2"])
self.assertIsInstance(check, string_types)

def test_check_filter_specific_tag_bad(self):
"""Raise errors"""
with self.assertRaises(TypeError):
checker._check_filter_specific_tag(specific_tag="oh_yeah_i_m_a_keyword")


# #############################################################################
Expand Down
66 changes: 35 additions & 31 deletions tests/test_download_hosted.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@

# Standard library
import logging
from os import environ
from os import environ, path
from sys import exit
from tempfile import mkstemp
from tempfile import mkdtemp
import unittest

# module target
Expand Down Expand Up @@ -49,46 +49,50 @@ def tearDown(self):
"""Executed after each test."""
pass

# # metadata models
# def test_dl_hosted(self):
# """Download an hosted data from Isogeo metadata."""
# search = self.isogeo.search(self.bearer, whole_share=0,
# query="action:download type:dataset",
# sub_resources=["links", ],
# page_size=100)
# # get an hosted link
# for md in search.get("results"):
# for link in md.get("links"):
# if link.get("type") == "hosted":
# target_link = link
# else:
# continue

# # download the XML version
# print(target_link)
# dl_stream = self.isogeo.dl_hosted(self.bearer,
# id_resource=md.get("_id"),
# resource_link=target_link)

# # create tempfile and fill with downloaded XML
# # tmp_output = mkstemp(prefix="IsogeoPySDK_" + md.get("_id")[:5])
# with open(dl_stream[1], 'wb') as fd:
# for block in dl_stream[0].iter_content(1024):
# fd.write(block)
def test_dl_hosted(self):
"""Download an hosted data from Isogeo metadata."""
search = self.isogeo.search(self.bearer, whole_share=0,
query="action:download type:dataset",
sub_resources=["links", ],
page_size=100)
# get an hosted link
for md in search.get("results"):
for link in md.get("links"):
if link.get("type") == "hosted":
target_link = link
else:
continue

# stream hosted data
# Example of link dict:
# {
# "_id": "c7725558b34a4ea8bfe475ca19e27641",
# "type": "hosted",
# "title": "bootstrap-4.0.0.zip",
# "url": "/resources/b765d9886f4b4fc69df65e5206f39a9d/links/c7725558b34a4ea8bfe475ca19e27641.bin",
# "kind": "data",
# "actions": ["download", ],
# "size": "2253029",
# }
dl_stream = self.isogeo.dl_hosted(self.bearer,
resource_link=target_link)

# create tempfile and fill with downloaded XML
tmp_output = mkdtemp(prefix="IsogeoPySDK_")
with open(path.join(tmp_output, dl_stream[1]), 'wb') as fd:
for block in dl_stream[0].iter_content(1024):
fd.write(block)

def test_dl_hosted_bad(self):
"""Test errors raised by download method"""
with self.assertRaises(ValueError):
self.isogeo.dl_hosted(self.bearer,
id_resource="trust_me_its_an_uuid",
resource_link={})
with self.assertRaises(TypeError):
self.isogeo.dl_hosted(self.bearer,
id_resource="ff40128027344640a19cc60af0c5c1d1",
resource_link="my_resource_link_is_a_nice_string")
with self.assertRaises(ValueError):
self.isogeo.dl_hosted(self.bearer,
id_resource="ff40128027344640a19cc60af0c5c1d1",
resource_link={"type": "url", }
)

Expand Down

0 comments on commit 0c4c5fb

Please sign in to comment.