When time traveling, *Quality-time* would show deleted reports after …

…their deletion date. (#2998)

* When time traveling, *Quality-time* would show deleted reports after their deletion date. Fixes #2997.

components/server/src/database/reports.py

@@ -21,18 +21,17 @@
 def latest_reports(database: Database, data_model: dict, max_iso_timestamp: str = "") -> list[Report]:
     """Return the latest, undeleted, reports in the reports collection."""
     if max_iso_timestamp and max_iso_timestamp < iso_timestamp():
-        report_filter = dict(deleted=DOES_NOT_EXIST, timestamp={"$lt": max_iso_timestamp})
+        report_filter = dict(timestamp={"$lt": max_iso_timestamp})
         report_uuids = database.reports.distinct("report_uuid", report_filter)
-        reports = []
+        report_dicts = []
         for report_uuid in report_uuids:
             report_filter["report_uuid"] = report_uuid
             report_dict = database.reports.find_one(report_filter, sort=TIMESTAMP_DESCENDING)
-            report = Report(data_model, report_dict)
-            reports.append(report)
+            if "deleted" not in report_dict:
+                report_dicts.append(report_dict)
     else:
         report_dicts = database.reports.find({"last": True, "deleted": DOES_NOT_EXIST})
-        reports = [Report(data_model, report_dict) for report_dict in report_dicts]
-    return reports
+    return [Report(data_model, report_dict) for report_dict in report_dicts]
 
 
 def latest_report(database: Database, data_model, report_uuid: str) -> Report:

components/server/tests/routes/external/test_report.py

@@ -21,22 +21,38 @@
 from server_utilities.functions import asymmetric_encrypt
 from server_utilities.type import ReportId
 
-from ...fixtures import JENNY, JOHN, METRIC_ID, REPORT_ID, REPORT_ID2, SOURCE_ID, SUBJECT_ID, create_report
+from ...fixtures import JENNY, METRIC_ID, REPORT_ID, REPORT_ID2, SOURCE_ID, SUBJECT_ID, create_report
 
 
-@patch("bottle.request")
-class PostReportAttributeTest(unittest.TestCase):
-    """Unit tests for the post report attribute route."""
+class ReportTestCase(unittest.TestCase):  # skipcq: PTC-W0046
+    """Base class for report route unit tests."""
 
     def setUp(self):
         """Override to set up a database with a report and a user session."""
         self.database = Mock()
-        self.report = Report({}, dict(_id="id", report_uuid=REPORT_ID, title="Title"))
+        self.database.sessions.find_one.return_value = JENNY
+        self.database.datamodels.find_one.return_value = dict(
+            _id="id",
+            scales=["count", "percentage"],
+            subjects=dict(subject_type=dict(name="Subject type")),
+            metrics=dict(
+                metric_type=dict(
+                    name="Metric type",
+                    scales=["count", "percentage"],
+                )
+            ),
+            sources=dict(source_type=dict(name="Source type", parameters={"url": {"type": "not a password"}})),
+        )
+        self.report = Report({}, create_report())
         self.database.reports.find.return_value = [self.report]
-        self.database.sessions.find_one.return_value = JOHN
-        self.database.datamodels.find_one.return_value = {}
+        self.database.reports.find_one.return_value = self.report
         self.database.measurements.find.return_value = []
 
+
+@patch("bottle.request")
+class PostReportAttributeTest(ReportTestCase):
+    """Unit tests for the post report attribute route."""
+
     def test_post_report_title(self, request):
         """Test that the report title can be changed."""
         request.json = dict(title="New title")
@@ -46,8 +62,8 @@ def test_post_report_title(self, request):
         self.assertEqual(
             dict(
                 uuids=[REPORT_ID],
-                email=JOHN["email"],
-                description="John changed the title of report 'Title' from 'Title' to 'New title'.",
+                email=JENNY["email"],
+                description="Jenny changed the title of report 'Report' from 'Report' to 'New title'.",
             ),
             updated_report["delta"],
         )
@@ -59,39 +75,34 @@ def test_post_report_layout(self, request):
         self.database.reports.insert_one.assert_called_once_with(self.report)
         updated_report = self.database.reports.insert_one.call_args[0][0]
         self.assertEqual(
-            dict(uuids=[REPORT_ID], email=JOHN["email"], description="John changed the layout of report 'Title'."),
+            dict(uuids=[REPORT_ID], email=JENNY["email"], description="Jenny changed the layout of report 'Report'."),
             updated_report["delta"],
         )
 
 
 @patch("bottle.request")
-class ReportIssueTrackerTest(unittest.TestCase):
+class ReportIssueTrackerTest(ReportTestCase):
     """Unit tests for the post report issue tracker attribute route."""
 
-    def setUp(self):
-        """Override to set up a database with a report and a user session."""
-        self.database = Mock()
-        self.report = Report({}, dict(_id="id", report_uuid=REPORT_ID, title="Title"))
-        self.database.reports.find.return_value = [self.report]
-        self.database.sessions.find_one.return_value = JOHN
-        self.database.datamodels.find_one.return_value = {}
-        self.database.measurements.find.return_value = []
-
     def test_post_report_issue_tracker_type(self, request):
         """Test that the issue tracker type can be changed."""
-        request.json = dict(type="jira")
+        request.json = dict(type="azure_devops")
         self.assertEqual(dict(ok=True), post_report_issue_tracker_attribute(REPORT_ID, "type", self.database))
         self.database.reports.insert_one.assert_called_once_with(self.report)
         updated_report = self.database.reports.insert_one.call_args[0][0]
         self.assertEqual(
             dict(
                 uuids=[REPORT_ID],
-                email=JOHN["email"],
-                description="John changed the type of the issue tracker of report 'Title' from '' to 'jira'.",
+                email=JENNY["email"],
+                description="Jenny changed the type of the issue tracker of report 'Report' from "
+                "'jira' to 'azure_devops'.",
             ),
             updated_report["delta"],
         )
-        self.assertEqual(dict(type="jira"), updated_report["issue_tracker"])
+        self.assertEqual(
+            dict(type="azure_devops", parameters=dict(username="jadoe", password="secret")),
+            updated_report["issue_tracker"],
+        )
 
     def test_post_report_issue_tracker_url(self, request):
         """Test that the issue tracker url can be changed."""
@@ -105,8 +116,8 @@ def test_post_report_issue_tracker_url(self, request):
         self.assertEqual(
             dict(
                 uuids=[REPORT_ID],
-                email=JOHN["email"],
-                description="John changed the url of the issue tracker of report 'Title' from '' to 'https://jira'.",
+                email=JENNY["email"],
+                description="Jenny changed the url of the issue tracker of report 'Report' from '' to 'https://jira'.",
             ),
             updated_report["delta"],
         )
@@ -121,28 +132,35 @@ def test_post_report_issue_tracker_username(self, request):
         self.assertEqual(
             dict(
                 uuids=[REPORT_ID],
-                email=JOHN["email"],
-                description="John changed the username of the issue tracker of report 'Title' from '' to 'jodoe'.",
+                email=JENNY["email"],
+                description="Jenny changed the username of the issue tracker of report 'Report' from "
+                "'jadoe' to 'jodoe'.",
             ),
             updated_report["delta"],
         )
-        self.assertEqual(dict(parameters=dict(username="jodoe")), updated_report["issue_tracker"])
+        self.assertEqual(
+            dict(parameters=dict(username="jodoe", password="secret"), type="jira"), updated_report["issue_tracker"]
+        )
 
     def test_post_report_issue_tracker_password(self, request):
         """Test that the issue tracker password can be changed."""
-        request.json = dict(password="secret")
+        request.json = dict(password="another secret")
         self.assertEqual(dict(ok=True), post_report_issue_tracker_attribute(REPORT_ID, "password", self.database))
         self.database.reports.insert_one.assert_called_once_with(self.report)
         updated_report = self.database.reports.insert_one.call_args[0][0]
         self.assertEqual(
             dict(
                 uuids=[REPORT_ID],
-                email=JOHN["email"],
-                description="John changed the password of the issue tracker of report 'Title' from '' to '******'.",
+                email=JENNY["email"],
+                description="Jenny changed the password of the issue tracker of report 'Report' from "
+                "'******' to '**************'.",
             ),
             updated_report["delta"],
         )
-        self.assertEqual(dict(parameters=dict(password="secret")), updated_report["issue_tracker"])
+        self.assertEqual(
+            dict(parameters=dict(password="another secret", username="jadoe"), type="jira"),
+            updated_report["issue_tracker"],
+        )
 
     def test_post_report_issue_tracker_password_unchanged(self, request):
         """Test that nothing happens when the new issue tracker password is unchanged."""
@@ -152,55 +170,15 @@ def test_post_report_issue_tracker_password_unchanged(self, request):
         self.database.reports.insert_one.assert_not_called()
 
 
-class ReportTest(unittest.TestCase):
+class ReportTest(ReportTestCase):
     """Unit tests for adding, deleting, and getting reports."""
 
-    def setUp(self):
-        """Override to set up a database with a report and a user session."""
-        self.private_key = """-----BEGIN PRIVATE KEY-----
-MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBANdJVRdylaadsaau
-hRxNToIUIk/nSKMzfjjjP/20FEShkax1g4CYTwTdSMcuV+4blzzFSE+eDmMs1LNk
-jAPzfNAnHwJsjz2vt16JXDma+PuIPTCI5uobCbPUJty+6XlnzFyVjy36+SgeA8SM
-HHTprOxhwxU++O5cnzO7Jb4mjoOvAgMBAAECgYEAr9gMErzbE16Wroi53OYgDAua
-Ax3srLDwllK3/+fI7k3yCKVrpevCDz0XpulplOkgXNjfOXjmU4dYrLahztBgzrwt
-KzA7H8XylleIbuk7wUJ8jD+1dzxgu/ZB+iLzUla8r9/MmdhAzELmYBc9hIEWl6FW
-2BlQxmLNbOj2kh/aWoECQQD4GyLDzxEFVBPYYo+Ut3T05a0IlCnCSKU6saDSuFFG
-GhiM1HQMAnnuC3okgVpAOA7Rn2z9xMqLcdiv+Amnzh3hAkEA3iLgQUwMj6v97Jkb
-KFxQazzkOmgMKFGH2MbZGGwDDva1QlD9awjBW0aj4nUHNsUob6LVJCbCocQFSNDu
-eXgzjwJATSg7NoPFuk98YHW+SzSGZcarehiBqA7pe4hUCFQTymZBLkK/2CBJBPOC
-x6mGhKQqT5xxy7WQe68rAQZ1Ej9yYQJAbgd8aRuQRUH+HsmfyBghxVx99+g9zWLF
-FT05n30w7qKJGfYf8Hp/vAR7fNpW3mw+IT3YsXV5hsMfkvfah9RgRQJAVGysMIfp
-eX94CsogDhIWSaXreAfpcWQu1Dg5FCmpZTGRJps2x52CPq5icgBZeIODElIvkJbn
-JqqQtg8ZsTm6Pw==
------END PRIVATE KEY-----
-"""
-
-        self.public_key = """-----BEGIN PUBLIC KEY-----
-MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXSVUXcpWmnbGmroUcTU6CFCJP
-50ijM3444z/9tBREoZGsdYOAmE8E3UjHLlfuG5c8xUhPng5jLNSzZIwD83zQJx8C
-bI89r7deiVw5mvj7iD0wiObqGwmz1Cbcvul5Z8xclY8t+vkoHgPEjBx06azsYcMV
-PvjuXJ8zuyW+Jo6DrwIDAQAB
------END PUBLIC KEY-----
-"""
+    PAST_DATE = "2021-08-31T23:59:59.000Z"
 
-        self.database = Mock()
-        self.database.sessions.find_one.return_value = JENNY
-        self.database.datamodels.find_one.return_value = dict(
-            _id="id",
-            scales=["count", "percentage"],
-            subjects=dict(subject_type=dict(name="Subject type")),
-            metrics=dict(
-                metric_type=dict(
-                    name="Metric type",
-                    scales=["count", "percentage"],
-                )
-            ),
-            sources=dict(source_type=dict(name="Source type", parameters={"url": {"type": "not a password"}})),
-        )
-        self.report = create_report()
-        self.database.reports.find.return_value = [self.report]
-        self.database.secrets.find_one.return_value = {"public_key": self.public_key, "private_key": self.private_key}
-        self.database.measurements.find.return_value = []
+    def setUp(self):
+        """Extend to set up a database with a report and a user session."""
+        super().setUp()
+        self.database.reports.distinct.return_value = [REPORT_ID]
         self.database.measurements.find_one.return_value = {"sources": []}
         self.database.measurements.aggregate.return_value = []
         self.options = (
@@ -212,6 +190,23 @@ def test_get_report(self):
         """Test that a report can be retrieved."""
         self.assertEqual(REPORT_ID, get_report(self.database, REPORT_ID)["reports"][0][REPORT_ID])
 
+    def test_get_all_reports(self):
+        """Test that all reports can be retrieved."""
+        self.assertEqual(1, len(get_report(self.database)["reports"]))
+
+    @patch("bottle.request")
+    def test_get_all_reports_with_time_travel(self, request):
+        """Test that all reports can be retrieved."""
+        request.query = dict(report_date=self.PAST_DATE)
+        self.assertEqual(1, len(get_report(self.database)["reports"]))
+
+    @patch("bottle.request")
+    def test_ignore_deleted_reports_when_time_traveling(self, request):
+        """Test that deleted reports are not retrieved."""
+        request.query = dict(report_date=self.PAST_DATE)
+        self.report["deleted"] = "true"
+        self.assertEqual(0, len(get_report(self.database)["reports"]))
+
     def test_get_report_and_info_about_other_reports(self):
         """Test that a report can be retrieved, and that other reports are also returned."""
         self.database.reports.find.return_value.insert(0, dict(_id="id2", report_uuid=REPORT_ID2))
@@ -225,9 +220,7 @@ def test_get_report_missing(self):
     @patch("bottle.request")
     def test_get_old_report(self, request):
         """Test that an old report can be retrieved and credentials are hidden."""
-        request.query = dict(report_date="2020-08-31T23:59:59.000Z")
-        self.database.reports.distinct.return_value = [REPORT_ID]
-        self.database.reports.find_one.return_value = create_report()
+        request.query = dict(report_date=self.PAST_DATE)
         report = get_report(self.database, REPORT_ID)["reports"][0]
         self.assertEqual("this string replaces credentials", report["issue_tracker"]["parameters"]["password"])
         self.assertEqual(
@@ -250,8 +243,7 @@ def test_issue_status(self):
         )
         self.database.measurements.aggregate.return_value = []
         self.database.measurements.find.return_value = [measurement]
-        self.database.reports.find.return_value = [report := create_report()]
-        report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["issue_ids"] = ["FOO-42"]
+        self.report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["issue_ids"] = ["FOO-42"]
         report = get_report(self.database, REPORT_ID)["reports"][0]
         self.assertEqual(issue_status, report["subjects"][SUBJECT_ID]["metrics"][METRIC_ID]["issue_status"][0])
 
@@ -376,6 +368,59 @@ def test_get_pdf_report(self, requests_get):
             f"http://renderer:9000/api/render?url=http%3A//www%3A80/{REPORT_ID}&{self.options}"
         )
 
+    @patch("requests.get")
+    def test_get_pdf_tag_report(self, requests_get):
+        """Test that a PDF version of a tag report can be retrieved."""
+        requests_get.return_value = Mock(content=b"PDF")
+        self.assertEqual(b"PDF", export_report_as_pdf(cast(ReportId, "tag-security")))
+        requests_get.assert_called_once_with(
+            f"http://renderer:9000/api/render?url=http%3A//www%3A80/tag-security&{self.options}"
+        )
+
+    def test_delete_report(self):
+        """Test that the report can be deleted."""
+        self.assertEqual(dict(ok=True), delete_report(REPORT_ID, self.database))
+        inserted = self.database.reports.insert_one.call_args_list[0][0][0]
+        self.assertEqual(
+            dict(uuids=[REPORT_ID], email=JENNY["email"], description="Jenny deleted the report 'Report'."),
+            inserted["delta"],
+        )
+
+
+class ReportImportAndExportTest(ReportTestCase):
+    """Unit tests for importing and exporting reports."""
+
+    def setUp(self):
+        """Extend to set up a database with a report and a user session."""
+        super().setUp()
+        self.private_key = """-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBANdJVRdylaadsaau
+hRxNToIUIk/nSKMzfjjjP/20FEShkax1g4CYTwTdSMcuV+4blzzFSE+eDmMs1LNk
+jAPzfNAnHwJsjz2vt16JXDma+PuIPTCI5uobCbPUJty+6XlnzFyVjy36+SgeA8SM
+HHTprOxhwxU++O5cnzO7Jb4mjoOvAgMBAAECgYEAr9gMErzbE16Wroi53OYgDAua
+Ax3srLDwllK3/+fI7k3yCKVrpevCDz0XpulplOkgXNjfOXjmU4dYrLahztBgzrwt
+KzA7H8XylleIbuk7wUJ8jD+1dzxgu/ZB+iLzUla8r9/MmdhAzELmYBc9hIEWl6FW
+2BlQxmLNbOj2kh/aWoECQQD4GyLDzxEFVBPYYo+Ut3T05a0IlCnCSKU6saDSuFFG
+GhiM1HQMAnnuC3okgVpAOA7Rn2z9xMqLcdiv+Amnzh3hAkEA3iLgQUwMj6v97Jkb
+KFxQazzkOmgMKFGH2MbZGGwDDva1QlD9awjBW0aj4nUHNsUob6LVJCbCocQFSNDu
+eXgzjwJATSg7NoPFuk98YHW+SzSGZcarehiBqA7pe4hUCFQTymZBLkK/2CBJBPOC
+x6mGhKQqT5xxy7WQe68rAQZ1Ej9yYQJAbgd8aRuQRUH+HsmfyBghxVx99+g9zWLF
+FT05n30w7qKJGfYf8Hp/vAR7fNpW3mw+IT3YsXV5hsMfkvfah9RgRQJAVGysMIfp
+eX94CsogDhIWSaXreAfpcWQu1Dg5FCmpZTGRJps2x52CPq5icgBZeIODElIvkJbn
+JqqQtg8ZsTm6Pw==
+-----END PRIVATE KEY-----
+"""
+
+        self.public_key = """-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXSVUXcpWmnbGmroUcTU6CFCJP
+50ijM3444z/9tBREoZGsdYOAmE8E3UjHLlfuG5c8xUhPng5jLNSzZIwD83zQJx8C
+bI89r7deiVw5mvj7iD0wiObqGwmz1Cbcvul5Z8xclY8t+vkoHgPEjBx06azsYcMV
+PvjuXJ8zuyW+Jo6DrwIDAQAB
+-----END PUBLIC KEY-----
+"""
+
+        self.database.secrets.find_one.return_value = {"public_key": self.public_key, "private_key": self.private_key}
+
     def test_get_json_report(self):
         """Test that a JSON version of the report can be retrieved with encrypted credentials."""
         expected_report = copy.deepcopy(self.report)
@@ -422,24 +467,6 @@ def test_get_json_report_with_public_key(self, request):
         self.assertTrue(isinstance(exported_password, tuple))
         self.assertTrue(len(exported_password) == 2)
 
-    @patch("requests.get")
-    def test_get_pdf_tag_report(self, requests_get):
-        """Test that a PDF version of a tag report can be retrieved."""
-        requests_get.return_value = Mock(content=b"PDF")
-        self.assertEqual(b"PDF", export_report_as_pdf(cast(ReportId, "tag-security")))
-        requests_get.assert_called_once_with(
-            f"http://renderer:9000/api/render?url=http%3A//www%3A80/tag-security&{self.options}"
-        )
-
-    def test_delete_report(self):
-        """Test that the report can be deleted."""
-        self.assertEqual(dict(ok=True), delete_report(REPORT_ID, self.database))
-        inserted = self.database.reports.insert_one.call_args_list[0][0][0]
-        self.assertEqual(
-            dict(uuids=[REPORT_ID], email=JENNY["email"], description="Jenny deleted the report 'Report'."),
-            inserted["delta"],
-        )
-
     @patch("bottle.request")
     def test_post_report_import(self, request):
         """Test that a report is imported correctly."""

docs/src/changelog.md

@@ -10,6 +10,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+### Fixed
+
+- When time traveling, *Quality-time* would show deleted reports after their deletion date. Fixes [#2997](https://github.com/ICTU/quality-time/issues/2997).
+
 ### Added
 
 - Support JMeter CSV output as source for the 'slow transactions' metric. Closes [#2966](https://github.com/ICTU/quality-time/issues/2966).