When time traveling, *Quality-time* would show deleted reports after …

…their deletion date. Fixes #2997.

components/server/src/database/reports.py

@@ -21,18 +21,17 @@
 def latest_reports(database: Database, data_model: dict, max_iso_timestamp: str = "") -> list[Report]:
     """Return the latest, undeleted, reports in the reports collection."""
     if max_iso_timestamp and max_iso_timestamp < iso_timestamp():
-        report_filter = dict(deleted=DOES_NOT_EXIST, timestamp={"$lt": max_iso_timestamp})
+        report_filter = dict(timestamp={"$lt": max_iso_timestamp})
         report_uuids = database.reports.distinct("report_uuid", report_filter)
-        reports = []
+        report_dicts = []
         for report_uuid in report_uuids:
             report_filter["report_uuid"] = report_uuid
             report_dict = database.reports.find_one(report_filter, sort=TIMESTAMP_DESCENDING)
-            report = Report(data_model, report_dict)
-            reports.append(report)
+            if "deleted" not in report_dict:
+                report_dicts.append(report_dict)
     else:
         report_dicts = database.reports.find({"last": True, "deleted": DOES_NOT_EXIST})
-        reports = [Report(data_model, report_dict) for report_dict in report_dicts]
-    return reports
+    return [Report(data_model, report_dict) for report_dict in report_dicts]
 
 
 def latest_report(database: Database, data_model, report_uuid: str) -> Report:

components/server/tests/routes/external/test_report.py

@@ -157,32 +157,6 @@ class ReportTest(unittest.TestCase):
 
     def setUp(self):
         """Override to set up a database with a report and a user session."""
-        self.private_key = """-----BEGIN PRIVATE KEY-----
-MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBANdJVRdylaadsaau
-hRxNToIUIk/nSKMzfjjjP/20FEShkax1g4CYTwTdSMcuV+4blzzFSE+eDmMs1LNk
-jAPzfNAnHwJsjz2vt16JXDma+PuIPTCI5uobCbPUJty+6XlnzFyVjy36+SgeA8SM
-HHTprOxhwxU++O5cnzO7Jb4mjoOvAgMBAAECgYEAr9gMErzbE16Wroi53OYgDAua
-Ax3srLDwllK3/+fI7k3yCKVrpevCDz0XpulplOkgXNjfOXjmU4dYrLahztBgzrwt
-KzA7H8XylleIbuk7wUJ8jD+1dzxgu/ZB+iLzUla8r9/MmdhAzELmYBc9hIEWl6FW
-2BlQxmLNbOj2kh/aWoECQQD4GyLDzxEFVBPYYo+Ut3T05a0IlCnCSKU6saDSuFFG
-GhiM1HQMAnnuC3okgVpAOA7Rn2z9xMqLcdiv+Amnzh3hAkEA3iLgQUwMj6v97Jkb
-KFxQazzkOmgMKFGH2MbZGGwDDva1QlD9awjBW0aj4nUHNsUob6LVJCbCocQFSNDu
-eXgzjwJATSg7NoPFuk98YHW+SzSGZcarehiBqA7pe4hUCFQTymZBLkK/2CBJBPOC
-x6mGhKQqT5xxy7WQe68rAQZ1Ej9yYQJAbgd8aRuQRUH+HsmfyBghxVx99+g9zWLF
-FT05n30w7qKJGfYf8Hp/vAR7fNpW3mw+IT3YsXV5hsMfkvfah9RgRQJAVGysMIfp
-eX94CsogDhIWSaXreAfpcWQu1Dg5FCmpZTGRJps2x52CPq5icgBZeIODElIvkJbn
-JqqQtg8ZsTm6Pw==
------END PRIVATE KEY-----
-"""
-
-        self.public_key = """-----BEGIN PUBLIC KEY-----
-MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXSVUXcpWmnbGmroUcTU6CFCJP
-50ijM3444z/9tBREoZGsdYOAmE8E3UjHLlfuG5c8xUhPng5jLNSzZIwD83zQJx8C
-bI89r7deiVw5mvj7iD0wiObqGwmz1Cbcvul5Z8xclY8t+vkoHgPEjBx06azsYcMV
-PvjuXJ8zuyW+Jo6DrwIDAQAB
------END PUBLIC KEY-----
-"""
-
         self.database = Mock()
         self.database.sessions.find_one.return_value = JENNY
         self.database.datamodels.find_one.return_value = dict(
@@ -199,7 +173,6 @@ def setUp(self):
         )
         self.report = create_report()
         self.database.reports.find.return_value = [self.report]
-        self.database.secrets.find_one.return_value = {"public_key": self.public_key, "private_key": self.private_key}
         self.database.measurements.find.return_value = []
         self.database.measurements.find_one.return_value = {"sources": []}
         self.database.measurements.aggregate.return_value = []
@@ -212,6 +185,27 @@ def test_get_report(self):
         """Test that a report can be retrieved."""
         self.assertEqual(REPORT_ID, get_report(self.database, REPORT_ID)["reports"][0][REPORT_ID])
 
+    def test_get_all_reports(self):
+        """Test that all reports can be retrieved."""
+        self.assertEqual(1, len(get_report(self.database)["reports"]))
+
+    @patch("bottle.request")
+    def test_get_all_reports_with_time_travel(self, request):
+        """Test that all reports can be retrieved."""
+        request.query = dict(report_date="2020-08-31T23:59:59.000Z")
+        self.database.reports.distinct.return_value = [REPORT_ID]
+        self.database.reports.find_one.return_value = create_report()
+        self.assertEqual(1, len(get_report(self.database)["reports"]))
+
+    @patch("bottle.request")
+    def test_ignore_deleted_reports_when_time_traveling(self, request):
+        """Test that deleted reports are not retrieved."""
+        request.query = dict(report_date="2020-08-31T23:59:59.000Z")
+        self.database.reports.distinct.return_value = [REPORT_ID]
+        self.database.reports.find_one.return_value = report = create_report()
+        report["deleted"] = "true"
+        self.assertEqual(0, len(get_report(self.database)["reports"]))
+
     def test_get_report_and_info_about_other_reports(self):
         """Test that a report can be retrieved, and that other reports are also returned."""
         self.database.reports.find.return_value.insert(0, dict(_id="id2", report_uuid=REPORT_ID2))
@@ -376,6 +370,74 @@ def test_get_pdf_report(self, requests_get):
             f"http://renderer:9000/api/render?url=http%3A//www%3A80/{REPORT_ID}&{self.options}"
         )
 
+    @patch("requests.get")
+    def test_get_pdf_tag_report(self, requests_get):
+        """Test that a PDF version of a tag report can be retrieved."""
+        requests_get.return_value = Mock(content=b"PDF")
+        self.assertEqual(b"PDF", export_report_as_pdf(cast(ReportId, "tag-security")))
+        requests_get.assert_called_once_with(
+            f"http://renderer:9000/api/render?url=http%3A//www%3A80/tag-security&{self.options}"
+        )
+
+    def test_delete_report(self):
+        """Test that the report can be deleted."""
+        self.assertEqual(dict(ok=True), delete_report(REPORT_ID, self.database))
+        inserted = self.database.reports.insert_one.call_args_list[0][0][0]
+        self.assertEqual(
+            dict(uuids=[REPORT_ID], email=JENNY["email"], description="Jenny deleted the report 'Report'."),
+            inserted["delta"],
+        )
+
+
+class ReportImportAndExportTest(unittest.TestCase):
+    """Unit tests for importing and exporting reports."""
+
+    def setUp(self):
+        """Override to set up a database with a report and a user session."""
+        self.private_key = """-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBANdJVRdylaadsaau
+hRxNToIUIk/nSKMzfjjjP/20FEShkax1g4CYTwTdSMcuV+4blzzFSE+eDmMs1LNk
+jAPzfNAnHwJsjz2vt16JXDma+PuIPTCI5uobCbPUJty+6XlnzFyVjy36+SgeA8SM
+HHTprOxhwxU++O5cnzO7Jb4mjoOvAgMBAAECgYEAr9gMErzbE16Wroi53OYgDAua
+Ax3srLDwllK3/+fI7k3yCKVrpevCDz0XpulplOkgXNjfOXjmU4dYrLahztBgzrwt
+KzA7H8XylleIbuk7wUJ8jD+1dzxgu/ZB+iLzUla8r9/MmdhAzELmYBc9hIEWl6FW
+2BlQxmLNbOj2kh/aWoECQQD4GyLDzxEFVBPYYo+Ut3T05a0IlCnCSKU6saDSuFFG
+GhiM1HQMAnnuC3okgVpAOA7Rn2z9xMqLcdiv+Amnzh3hAkEA3iLgQUwMj6v97Jkb
+KFxQazzkOmgMKFGH2MbZGGwDDva1QlD9awjBW0aj4nUHNsUob6LVJCbCocQFSNDu
+eXgzjwJATSg7NoPFuk98YHW+SzSGZcarehiBqA7pe4hUCFQTymZBLkK/2CBJBPOC
+x6mGhKQqT5xxy7WQe68rAQZ1Ej9yYQJAbgd8aRuQRUH+HsmfyBghxVx99+g9zWLF
+FT05n30w7qKJGfYf8Hp/vAR7fNpW3mw+IT3YsXV5hsMfkvfah9RgRQJAVGysMIfp
+eX94CsogDhIWSaXreAfpcWQu1Dg5FCmpZTGRJps2x52CPq5icgBZeIODElIvkJbn
+JqqQtg8ZsTm6Pw==
+-----END PRIVATE KEY-----
+"""
+
+        self.public_key = """-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXSVUXcpWmnbGmroUcTU6CFCJP
+50ijM3444z/9tBREoZGsdYOAmE8E3UjHLlfuG5c8xUhPng5jLNSzZIwD83zQJx8C
+bI89r7deiVw5mvj7iD0wiObqGwmz1Cbcvul5Z8xclY8t+vkoHgPEjBx06azsYcMV
+PvjuXJ8zuyW+Jo6DrwIDAQAB
+-----END PUBLIC KEY-----
+"""
+
+        self.database = Mock()
+        self.database.sessions.find_one.return_value = JENNY
+        self.database.datamodels.find_one.return_value = dict(
+            _id="id",
+            scales=["count", "percentage"],
+            subjects=dict(subject_type=dict(name="Subject type")),
+            metrics=dict(
+                metric_type=dict(
+                    name="Metric type",
+                    scales=["count", "percentage"],
+                )
+            ),
+            sources=dict(source_type=dict(name="Source type", parameters={"url": {"type": "not a password"}})),
+        )
+        self.report = create_report()
+        self.database.reports.find.return_value = [self.report]
+        self.database.secrets.find_one.return_value = {"public_key": self.public_key, "private_key": self.private_key}
+
     def test_get_json_report(self):
         """Test that a JSON version of the report can be retrieved with encrypted credentials."""
         expected_report = copy.deepcopy(self.report)
@@ -422,24 +484,6 @@ def test_get_json_report_with_public_key(self, request):
         self.assertTrue(isinstance(exported_password, tuple))
         self.assertTrue(len(exported_password) == 2)
 
-    @patch("requests.get")
-    def test_get_pdf_tag_report(self, requests_get):
-        """Test that a PDF version of a tag report can be retrieved."""
-        requests_get.return_value = Mock(content=b"PDF")
-        self.assertEqual(b"PDF", export_report_as_pdf(cast(ReportId, "tag-security")))
-        requests_get.assert_called_once_with(
-            f"http://renderer:9000/api/render?url=http%3A//www%3A80/tag-security&{self.options}"
-        )
-
-    def test_delete_report(self):
-        """Test that the report can be deleted."""
-        self.assertEqual(dict(ok=True), delete_report(REPORT_ID, self.database))
-        inserted = self.database.reports.insert_one.call_args_list[0][0][0]
-        self.assertEqual(
-            dict(uuids=[REPORT_ID], email=JENNY["email"], description="Jenny deleted the report 'Report'."),
-            inserted["delta"],
-        )
-
     @patch("bottle.request")
     def test_post_report_import(self, request):
         """Test that a report is imported correctly."""

docs/src/changelog.md

@@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+### Fixed
+
+- When time traveling, *Quality-time* would show deleted reports after their deletion date. Fixes [#2997](https://github.com/ICTU/quality-time/issues/2997).
+
 ### Added
 
 - Support JMeter CSV output as source for the 'slow transactions' metric. Closes [#2966](https://github.com/ICTU/quality-time/issues/2966).

tests/feature_tests/features/report.feature

@@ -139,29 +139,6 @@ Feature: report
       """
     Then the import failed
 
-  Scenario: time travel to the past, before the report existed
-    When the client creates a report
-    And the client enters a report date that's too old
-    Then the report does not exist
-
-  Scenario: time travel to the future
-    When the client creates a report
-    And the client enters a future report date
-    Then the report title is "New report"
-
-  Scenario: time travel to the past and back to the present
-    When the client creates a report
-    And the client creates a subject
-    And the client creates a metric
-    And the client creates a source
-    And the collector measures "0"
-    Then the metric latest_measurement.count.value is "0"
-    When the client enters a report date that's not too old
-    And the collector measures "100"
-    Then the metric latest_measurement.count.value is "0"
-    When the client resets the report date
-    Then the metric latest_measurement.count.value is "100"
-
   Scenario: get non-existent report
     When the client gets a non-existing report
     Then the report does not exist

tests/feature_tests/features/time_travel.feature

@@ -0,0 +1,34 @@
+Feature: time travel
+  Displaying old reports
+
+  Background: the client is logged in
+    Given a logged-in client
+
+  Scenario: time travel to the past, before the report existed
+    When the client creates a report
+    And the client enters a report date that's too old
+    Then the report does not exist
+
+  Scenario: time travel to the future
+    When the client creates a report
+    And the client enters a future report date
+    Then the report title is "New report"
+
+  Scenario: time travel to the past and back to the present
+    When the client creates a report
+    And the client creates a subject
+    And the client creates a metric
+    And the client creates a source
+    And the collector measures "0"
+    Then the metric latest_measurement.count.value is "0"
+    When the client enters a report date that's not too old
+    And the collector measures "100"
+    Then the metric latest_measurement.count.value is "0"
+    When the client resets the report date
+    Then the metric latest_measurement.count.value is "100"
+
+  Scenario: time travel to a time in the past after a report was deleted
+    When the client creates a report
+    And the client deletes the report
+    And the client enters a report date that's not too old
+    Then the report does not exist

tests/feature_tests/steps/report.py

@@ -65,7 +65,7 @@ def reset_report_date(context):
 @when("the client enters a report date that's not too old")
 def time_travel(context):
     """Set a time in the past, but after the report was created."""
-    time.sleep(1)  # Make sure the previously created report is older than the report date
+    time.sleep(1)  # Make sure the previous created report is older than the report date
     context.report_date = datetime.now(timezone.utc).replace(microsecond=0).isoformat()[: -len("+00:00")] + "Z"
     time.sleep(1)  # Make sure report date is in the past