Merge pull request #1356 from Cadair/database_filenames

Fix file names when downloading multiple files using database.download

CHANGELOG.md

@@ -23,6 +23,7 @@ Latest
  * Require JSOC request data calls have an email address attached.
  * Calculation of the solar rotation of a point on the Sun as seen from Earth, and its application to the de-rotation of mapcubes.
  * Downloaded files now keep file extensions rather than replacing all periods with underscores
+ * Fixed the downloading of files with duplicate names in sunpy.database
 
 0.5.0
 -----

sunpy/database/database.py

@@ -9,6 +9,7 @@
 import operator
 from datetime import datetime
 from contextlib import contextmanager
+import os.path
 
 from sqlalchemy import create_engine, exists
 from sqlalchemy.orm import sessionmaker
@@ -22,6 +23,11 @@
 from sunpy.net.attr import and_
 from sunpy.net.vso import VSOClient
 
+__authors__ = ['Simon Liedtke', 'Rajul Srivastava']
+__emails__ = [
+    'liedtke.simon@googlemail.com',
+    'rajul09@gmail.com'
+]
 
 class EntryNotFoundError(Exception):
     """This exception is raised if a database entry cannot be found by its
@@ -301,27 +307,33 @@ def _download_and_collect_entries(self, query_result, client=None,
             path=None, progress=False):
         if client is None:
             client = VSOClient()
-        for block in query_result:
-            paths = client.get([block], path).wait(progress=progress)
-            for path in paths:
-                qr_entry = tables.DatabaseEntry._from_query_result_block(block)
-                file_entries = list(
-                    tables.entries_from_file(path, self.default_waveunit))
-                for entry in file_entries:
-                    entry.source = qr_entry.source
-                    entry.provider = qr_entry.provider
-                    entry.physobs = qr_entry.physobs
-                    entry.fileid = qr_entry.fileid
-                    entry.observation_time_start =\
-                        qr_entry.observation_time_start
-                    entry.observation_time_end = qr_entry.observation_time_end
-                    entry.instrument = qr_entry.instrument
-                    entry.size = qr_entry.size
-                    entry.wavemin = qr_entry.wavemin
-                    entry.wavemax = qr_entry.wavemax
-                    entry.path = path
-                    entry.download_time = datetime.utcnow()
-                    yield entry
+
+        paths = client.get(query_result, path).wait(progress=progress)
+
+        for (path, block) in zip(paths, query_result):
+            qr_entry = tables.DatabaseEntry._from_query_result_block(block)
+
+            if os.path.isfile(path):
+                entries = tables.entries_from_file(path, self.default_waveunit)
+            elif os.path.isdir(path):
+                entries = tables.entries_from_dir(path, self.default_waveunit)
+            else:
+                raise ValueError('The path is neither a file nor directory')
+
+            for entry in entries:
+                entry.source = qr_entry.source
+                entry.provider = qr_entry.provider
+                entry.physobs = qr_entry.physobs
+                entry.fileid = qr_entry.fileid
+                entry.observation_time_start = qr_entry.observation_time_start
+                entry.observation_time_end = qr_entry.observation_time_end
+                entry.instrument = qr_entry.instrument
+                entry.size = qr_entry.size
+                entry.wavemin = qr_entry.wavemin
+                entry.wavemax = qr_entry.wavemax
+                entry.path = path
+                entry.download_time = datetime.utcnow()
+                yield entry
 
     def download(self, *query, **kwargs):
         """download(*query, client=sunpy.net.vso.VSOClient(), path=None, progress=False)

sunpy/database/tests/test_database.py

@@ -7,7 +7,9 @@
 
 import glob
 import ConfigParser
+import os
 import os.path
+import shutil
 import sys
 
 import pytest
@@ -27,7 +29,7 @@
 from sunpy.io import fits
 
 import sunpy.data.test
-import os
+
 testpath = sunpy.data.test.rootdir
 RHESSI_IMAGE = os.path.join(testpath, 'hsi_image_20101016_191218.fits')
 
@@ -784,6 +786,41 @@ def test_fetch(database, download_query, tmpdir):
     assert database[0].download_time == download_time
 
 
+@pytest.mark.online
+def test_fetch_separate_filenames():
+    # Setup
+    db = Database('sqlite:///')
+
+    download_query = [
+        vso.attrs.Time('2012-08-05', '2012-08-05 00:00:05'),
+        vso.attrs.Instrument('AIA')
+    ]
+
+    tmp_test_dir = os.path.join(
+        sunpy.config.get('downloads', 'download_dir'),
+        'tmp_test_dir/'
+    )
+
+    if not os.path.isdir(tmp_test_dir):
+        os.mkdir(tmp_test_dir)
+
+    path = tmp_test_dir + '{file}'
+
+    db.fetch(*download_query, path=path)
+
+    # Test
+    assert len(db) == 2
+
+    dir_contents = os.listdir(tmp_test_dir)
+    assert 'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits' in dir_contents
+    assert 'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits' in dir_contents
+    assert os.path.isfile(os.path.join(tmp_test_dir, 'aia_lev1_335a_2012_08_05t00_00_02_62z_image_lev1.fits'))
+    assert os.path.isfile(os.path.join(tmp_test_dir, 'aia_lev1_94a_2012_08_05t00_00_01_12z_image_lev1.fits'))
+
+    # Teardown
+    shutil.rmtree(tmp_test_dir)
+
+
 @pytest.mark.online
 def test_disable_undo(database, download_query, tmpdir):
     entry = DatabaseEntry()

sunpy/net/download.py

@@ -58,6 +58,7 @@ def _start_download(self, url, path, callback, errback):
                         else:
                             fd.write(rec)
         except Exception, e:
+            # TODO: Fix the silent failing
             if errback is not None:
                 with self.mutex:
                     self._close(errback, [e], server)

sunpy/net/vso/vso.py

@@ -600,6 +600,8 @@ def get(self, query_response, path=None, methods=('URL-FILE',), downloader=None,
         if path is None:
             path = os.path.join(config.get('downloads','download_dir'),
                                 '{file}')
+        path = os.path.expanduser(path)
+
         fileids = VSOClient.by_fileid(query_response)
         if not fileids:
             res.poke()