Merge 0f37da9 into 9d2577c

geospaas/catalog/models.py

@@ -195,6 +195,8 @@ def save(self, *args, **kwargs):
         # (apparently) good... see the test..
         #self.full_clean()
         # Check that the uri exists?
+        # TODO: check in the get_or_create method - valid uri-schemes are provided at
+        # https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
         super(DatasetURI, self).save(*args, **kwargs)
 
 class DatasetRelationship(models.Model):

geospaas/nansat_ingestor/managers.py

@@ -20,7 +20,7 @@
 
 class DatasetManager(models.Manager):
 
-    def get_or_create(self, uri, n_points=10, uri_filter_args={}, *args, **kwargs):
+    def get_or_create(self, uri, n_points=10, uri_filter_args=None, *args, **kwargs):
         ''' Create dataset and corresponding metadata
 
         Parameters:
@@ -36,6 +36,9 @@ def get_or_create(self, uri, n_points=10, uri_filter_args={}, *args, **kwargs):
         -------
             dataset and flag
         '''
+        if not uri_filter_args:
+            uri_filter_args = {}
+
         # Validate uri - this should fail if the uri doesn't point to a valid
         # file or stream
         valid_uri = validate_uri(uri)

geospaas/tests.py

@@ -0,0 +1,30 @@
+import os
+from mock import patch
+
+from django.test import TestCase
+
+from geospaas.utils import utils
+
+class TestUtils(TestCase):
+
+    def test_validate_uri_opendap(self):
+        uri = 'http://www.ifremer.fr/opendap/cerdap1/globcurrent/' \
+                'v2.0/global_012_deg/geostrophic/2014/001/' \
+                '20140101000000-GLOBCURRENT-L4-CURgeo_0m-ALT_OI-v02.0-fv01.0.nc'
+        self.assertTrue(utils.validate_uri(uri))
+
+    def test_validate_uri_opendap_does_not_exist(self):
+        uri = 'http://www.ifremer.fr/opendap/cerdap1/globcurrent/' \
+                'v2.0/global_012_deg/geostrophic/2014/001/' \
+                '20140101000000-GLOBCURRENT-L4-CURgeo_0m-ALT_OI-v02.0-fv01.0.nc.tull'
+        self.assertFalse(utils.validate_uri(uri))
+
+    @patch('os.path.isfile')
+    def test_validate_uri_local(self, mock_isfile):
+        mock_isfile.return_value = True
+        uri = 'file://localhost/some/folder/filename.ext'
+        self.assertTrue(utils.validate_uri(uri))
+
+    def test_validate_uri_local_does_not_exist(self):
+        uri = 'file://localhost/some/folder/filename.ext'
+        self.assertFalse(utils.validate_uri(uri))

geospaas/utils/utils.py

@@ -42,24 +42,27 @@ def product_path(module, filename):
     return path(module, filename, settings.PRODUCTS_ROOT)
 
 def validate_uri(uri):
-    """ Validation of URI:
+    """ Validation of URI and its existence
+
     URI should be either: file://localhost/some/path/filename.ext
            or accessible: http://www.eee.rrr/some/path
 
     If URI is not valid, the function rasies a ValueError or urrlib error
+
     """
+    validation_result = False
     uri_parts = urlparse(uri)
-    if uri_parts.scheme == 'file' and uri_parts.netloc == 'localhost' and len(uri_parts.path) > 0:
-        return True
+    if uri_parts.scheme=='file' and uri_parts.netloc=='localhost':
+        if os.path.isfile(uri_parts.path):
+            validation_result = True
     else:
-        raise ValueError('Invalid URI: %s' % uri)
-
-    if URLLIB_VERSION == 2:
-        request = urllibN.Request(uri)
-    else:
-        request = urllibN.PoolManager().request('GET', uri)
-
-    return True
+        if URLLIB_VERSION == 2:
+            request = urllibN.Request(uri)
+        else:
+            request = urllibN.PoolManager().request('GET', uri)
+        if request.status==200:
+            validation_result = True
+    return validation_result
 
 def nansat_filename(uri):
     # Check if data should be read as stream or as file? Or just: