Add option to get only part of the timeseries data

doc/dev/database.rst

@@ -663,7 +663,7 @@ Time series and related models
       pd2hts as `text format`_.
 
       Usually you don't need to access this file directly; instead, use
-      methods :meth:`~enhydris.hcore.models.Timeseries.get_all_data`,
+      methods :meth:`~enhydris.hcore.models.Timeseries.get_data`,
       :meth:`~enhydris.hcore.models.Timeseries.set_data`,
       :meth:`~enhydris.hcore.models.Timeseries.append_data`,
       :meth:`~enhydris.hcore.models.Timeseries.get_first_line` and
@@ -683,9 +683,12 @@ Time series and related models
       :attr:`~enhydris.hcore.models.Timeseries.datafile`, you must
       subsequently call :meth:`save()` to update these fields.
 
-   .. method:: enhydris.hcore.models.Timeseries.get_all_data()
+   .. method:: enhydris.hcore.models.Timeseries.get_data(start_date=None, end_date=None)
 
-      Return all data of the file in a pandas DataFrame.
+      Return the data of the file in a pandas DataFrame. If *start_date* or
+      *end_date* are specified, only this part of the data is returned.
+
+      An old, deprecated synonym for this method is ``get_all_data()``.
 
    .. method:: enhydris.hcore.models.Timeseries.set_data(data)
 

enhydris/api/tests.py

@@ -257,7 +257,7 @@ def testUploadTsDataUnauthenticated(self):
             encode_multipart(BOUNDARY,
                              {'timeseries_records': '2012-11-06 18:17,20,\n'}),
             content_type=MULTIPART_CONTENT)
-        t = self.timeseries1.get_all_data()
+        t = self.timeseries1.get_data()
         self.assertEqual(response.status_code, 403)
         self.assertEqual(len(t), 0)
 
@@ -271,7 +271,7 @@ def testUploadTsDataAsWrongUser(self):
             encode_multipart(BOUNDARY,
                              {'timeseries_records': '2012-11-06 18:17,20,\n'}),
             content_type=MULTIPART_CONTENT)
-        adataframe = self.timeseries1.get_all_data()
+        adataframe = self.timeseries1.get_data()
         self.assertEqual(response.status_code, 403)
         self.assertEqual(len(adataframe), 0)
         self.client.logout()
@@ -285,7 +285,7 @@ def testUploadTsDataGarbage(self):
             encode_multipart(BOUNDARY,
                              {'timeseries_records': '2012-aa-06 18:17,20,\n'}),
             content_type=MULTIPART_CONTENT)
-        adataframe = self.timeseries1.get_all_data()
+        adataframe = self.timeseries1.get_data()
         self.assertEqual(response.status_code, 400)
         self.assertEqual(len(adataframe), 0)
         self.client.logout()
@@ -302,7 +302,7 @@ def testUploadTsData(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content.decode('utf-8'), '1')
         adataframe = models.Timeseries.objects.get(pk=self.timeseries1.id
-                                                   ).get_all_data()
+                                                   ).get_data()
         self.assertEqual(len(adataframe), 1)
         self.assertEqual(adataframe.index[0], datetime(2012, 11, 6, 18, 17, 0))
         self.assertEqual(adataframe.iloc[0].value, 20)
@@ -320,7 +320,7 @@ def testUploadTsData(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content.decode('utf-8'), '2')
         adataframe = models.Timeseries.objects.get(pk=self.timeseries1.id
-                                                   ).get_all_data()
+                                                   ).get_data()
         self.assertEqual(len(adataframe), 3)
         self.assertEqual(adataframe.index[0],
                          datetime(2012, 11, 6, 18, 17, 0))
@@ -343,8 +343,7 @@ def testUploadTsData(self):
             content_type=MULTIPART_CONTENT)
         self.client.logout()
         self.assertEqual(response.status_code, 400)
-        t = models.Timeseries.objects.get(pk=self.timeseries1.id
-                                          ).get_all_data()
+        t = models.Timeseries.objects.get(pk=self.timeseries1.id).get_data()
         self.assertEqual(len(t), 3)
         self.client.logout()
 

enhydris/api/views.py

@@ -58,7 +58,7 @@ def get(self, request, pk, format=None):
         timeseries = models.Timeseries.objects.get(pk=int(pk))
         self.check_object_permissions(request, timeseries)
         response = HttpResponse(content_type='text/plain')
-        pd2hts.write(timeseries.get_all_data(), response)
+        pd2hts.write(timeseries.get_data(), response)
         return response
 
     def put(self, request, pk, format=None):

enhydris/hcore/models.py

@@ -765,14 +765,16 @@ def set_extra_timeseries_properties(self, adataframe):
         adataframe.location = location
         adataframe.comment = '%s\n\n%s' % (self.gentity.name, self.remarks)
 
-    def get_all_data(self):
+    def get_data(self, start_date=None, end_date=None):
         if self.datafile:
-            with open(self.datafile.path, 'r') as f:
-                result = pd2hts.read(f)
+            with open(self.datafile.path, 'r', newline='\n') as f:
+                result = pd2hts.read(f, start_date=start_date,
+                                     end_date=end_date)
                 self.set_extra_timeseries_properties(result)
         else:
             result = pd.DataFrame()
         return result
+    get_all_data = get_data
 
     def set_data(self, data):
         adataframe = pd2hts.read(data)

enhydris/hcore/tests/test_models.py

@@ -26,12 +26,32 @@ def setUp(self):
             """)))
 
     def test_get_all_data(self):
-        adataframe = self.ts_komboti_rain.get_all_data()
+        adataframe = self.ts_komboti_rain.get_data()
         self.assertEqual(len(adataframe), 3)
         self.assertAlmostEqual(adataframe.ix['2015-10-22 15:00'].value, 0)
         self.assertAlmostEqual(adataframe.ix['2015-10-22 15:10'].value, 0.1)
         self.assertAlmostEqual(adataframe.ix['2015-10-22 15:20'].value, 0.2)
 
+    def test_get_data_with_start_date(self):
+        adataframe = self.ts_komboti_rain.get_data(
+            start_date='2015-10-22 15:10')
+        self.assertEqual(len(adataframe), 2)
+        self.assertAlmostEqual(adataframe.ix['2015-10-22 15:10'].value, 0.1)
+        self.assertAlmostEqual(adataframe.ix['2015-10-22 15:20'].value, 0.2)
+
+    def test_get_data_with_end_date(self):
+        adataframe = self.ts_komboti_rain.get_data(
+            end_date=datetime(2015, 10, 22, 15, 10))
+        self.assertEqual(len(adataframe), 2)
+        self.assertAlmostEqual(adataframe.ix['2015-10-22 15:00'].value, 0)
+        self.assertAlmostEqual(adataframe.ix['2015-10-22 15:10'].value, 0.1)
+
+    def test_get_data_with_start_and_end_date(self):
+        adataframe = self.ts_komboti_rain.get_data(
+            start_date='2015-10-22 15:05', end_date='2015-10-22 15:15')
+        self.assertEqual(len(adataframe), 1)
+        self.assertAlmostEqual(adataframe.ix['2015-10-22 15:10'].value, 0.1)
+
     def test_start_date_end_date(self):
         atzinfo = timezone(timedelta(minutes=120), 'EET')
         start_date = datetime(2015, 10, 22, 15, 0, tzinfo=atzinfo)

enhydris/hcore/tests/test_views.py

@@ -434,7 +434,7 @@ def test_timeseries_data(self):
         self.assertTrue(form.is_valid())
         ts = form.save()
         ts.save()
-        self.assertEqual(len(ts.get_all_data()), 12872)
+        self.assertEqual(len(ts.get_data()), 12872)
 
         # Download
 

enhydris/hcore/views.py

@@ -917,7 +917,7 @@ def download_timeseries(request, object_id, start_date=None, end_date=None):
     if end_date:
         end_date = end_date.replace(tzinfo=None)
 
-    adataframe = timeseries.get_all_data()[start_date:end_date]
+    adataframe = timeseries.get_data(start_date=start_date, end_date=end_date)
     timeseries.set_extra_timeseries_properties(adataframe)
     response = HttpResponse(
         content_type='text/vnd.openmeteo.timeseries; charset=utf-8')

setup.py

@@ -17,7 +17,7 @@
     "django-bootstrap3>=5.1,<6",
     "simpletail>=0.1.2",
     "iso8601",
-    "pd2hts",
+    "pd2hts>=0.2",
     "pytz",
 ]