add update-ready-diffs daily covering the last week (#15)

.vps/systemd/copypatrol-backend-update-ready-diffs.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=copypatrol backend update ready diffs daemon
+After=network.target
+
+[Service]
+User=www-data
+Group=www-data
+EnvironmentFile=/etc/copypatrol-env.sh
+ExecStart=/var/www/.venv/bin/copypatrol-backend update-ready-diffs -log:update-ready-diffs.log
+ExecStopPost=/bin/sh -c 'if [ "$$EXIT_STATUS" != 0 ]; then python3 /var/www/copypatrol-backend/.vps/bin/failure-mailer.py %n %H; fi'
+SyslogIdentifier=copypatrol-backend-update-ready-diffs
+
+[Install]
+WantedBy=multi-user.target

.vps/systemd/copypatrol-backend-update-ready-diffs.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=copypatrol backend update ready diffs timer
+Requires=copypatrol-backend-update-ready-diffs.service
+
+[Timer]
+AccuracySec=1 hour
+OnStartupSec=1 hour
+OnUnitInactiveSec=1 day
+Unit=copypatrol-backend-update-ready-diffs.service
+
+[Install]
+WantedBy=timers.target

pyproject.toml

@@ -73,7 +73,7 @@ license = "MIT"
 name = "copypatrol-backend"
 readme = "README.md"
 repository = "https://github.com/JJMC89/copypatrol-backend"
-version = "2024.4.11"
+version = "2024.4.15"
 
 [tool.poetry.dependencies]
 python = ">=3.11,<4"

src/copypatrol_backend/cli.py

@@ -50,8 +50,9 @@ def _check_diff(
 def check_changes(*, poolsize: int = 1, limit: int | None = None) -> None:
     sessionmaker = database.create_sessionmaker()
     with sessionmaker.begin() as session:
-        diffs = database.queued_diffs_by_status(
+        diffs = database.diffs_by_status(
             session,
+            database.QueuedDiff,
             [database.Status.UNSUBMITTED, database.Status.CREATED],
             limit=limit,
         )
@@ -103,8 +104,9 @@ def check_changes(*, poolsize: int = 1, limit: int | None = None) -> None:
 def generate_reports(*, delta: datetime.timedelta | None = None) -> None:
     api = tca.TurnitinCoreAPI()
     with database.create_sessionmaker().begin() as session:
-        for diff in database.queued_diffs_by_status(
+        for diff in database.diffs_by_status(
             session,
+            database.QueuedDiff,
             database.Status.UPLOADED,
             delta=delta,
         ):
@@ -120,8 +122,9 @@ def generate_reports(*, delta: datetime.timedelta | None = None) -> None:
 def check_reports(*, delta: datetime.timedelta | None = None) -> None:
     api = tca.TurnitinCoreAPI()
     with database.create_sessionmaker().begin() as session:
-        for diff in database.queued_diffs_by_status(
+        for diff in database.diffs_by_status(
             session,
+            database.QueuedDiff,
             database.Status.PENDING,
             delta=delta,
         ):
@@ -134,6 +137,19 @@ def check_reports(*, delta: datetime.timedelta | None = None) -> None:
             api.handle_similarity_info(info=info, diff=diff)
 
 
+def update_ready_diffs(*, delta: datetime.timedelta | None = None) -> None:
+    with database.create_sessionmaker().begin() as session:
+        for diff in database.diffs_by_status(
+            session,
+            database.Diff,
+            database.Status.READY,
+            delta=delta,
+        ):
+            if diff.update_page() is None:
+                diff.status = database.Status.FIXED
+                diff.status_user_text = diff.site.username()
+
+
 def parse_script_args(*args: str) -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="copypatrol backend",
@@ -186,6 +202,13 @@ def parse_script_args(*args: str) -> argparse.Namespace:
         help=description,
         allow_abbrev=False,
     )
+    description = "update ready diffs for deletions and moves"
+    subparsers.add_parser(
+        "update-ready-diffs",
+        description=description,
+        help=description,
+        allow_abbrev=False,
+    )
     description = "setup database and (optionally) webhook"
     setup_subparser = subparsers.add_parser("setup", allow_abbrev=False)
     setup_subparser.add_argument(
@@ -207,6 +230,8 @@ def cli(*args: str) -> int:
         delta = datetime.timedelta(minutes=-30)
         check_reports(delta=delta)
         generate_reports(delta=delta)
+    elif parsed_args.arction == "update-ready-diffs":
+        update_ready_diffs(delta=datetime.timedelta(weeks=-1))
     elif parsed_args.action == "setup":
         database.create_tables()
         if parsed_args.webhook:

src/copypatrol_backend/database.py

@@ -3,7 +3,7 @@
 import datetime
 import os
 from enum import IntEnum
-from typing import TYPE_CHECKING, Self, Union
+from typing import TYPE_CHECKING, Self, TypeVar, Union
 from uuid import UUID
 
 import pywikibot
@@ -74,11 +74,14 @@
 
 
 class Status(IntEnum):
+    UNKNOWN = -99
     UNSUBMITTED = -4
     CREATED = -3
     UPLOADED = -2
     PENDING = -1
     READY = 0
+    FIXED = 1
+    NO_ACTION = 2
 
 
 class BinaryDecoratorBase(TypeDecorator[str]):
@@ -328,6 +331,9 @@ class Source(TableBase, kw_only=True):
     percent: Mapped[float] = mapped_column(UnsignedFloat)
 
 
+DiffT = TypeVar("DiffT", bound=DiffMixin)
+
+
 def add_revision(
     *,
     session: Session,
@@ -381,44 +387,45 @@ def create_tables() -> None:
     TableBase.metadata.create_all(_create_engine(echo=True), checkfirst=True)
 
 
-def queued_diff_from_submission_id(
-    session: Session,
-    submission_id: UUID,
-    /,
-) -> QueuedDiff | None:
-    stmt = select(QueuedDiff).where(QueuedDiff.submission_id == submission_id)
-    return session.scalars(stmt).one_or_none()
-
-
-def queued_diffs_by_status(
+def diffs_by_status(
     session: Session,
+    table_class: type[DiffT],
     status: Status | list[Status],
     /,
     *,
     delta: datetime.timedelta | None = None,
     limit: int | None = None,
-) -> Sequence[QueuedDiff]:
+) -> Sequence[DiffT]:
     if isinstance(status, Status):
         status = [status]
     if delta is None:
         delta = datetime.timedelta()
     stmt = (
-        select(QueuedDiff)
+        select(table_class)
         .where(
             and_(
-                QueuedDiff.status.in_(status),
-                QueuedDiff.status_timestamp <= Timestamp.utcnow() + delta,
+                table_class.status.in_(status),
+                table_class.status_timestamp <= Timestamp.utcnow() + delta,
             )
         )
-        .order_by(QueuedDiff.rev_timestamp.desc())
+        .order_by(table_class.rev_timestamp.desc())
         .limit(limit)
     )
     return session.scalars(stmt).all()
 
 
+def queued_diff_from_submission_id(
+    session: Session,
+    submission_id: UUID,
+    /,
+) -> QueuedDiff | None:
+    stmt = select(QueuedDiff).where(QueuedDiff.submission_id == submission_id)
+    return session.scalars(stmt).one_or_none()
+
+
 def diff_count(
     session: Session,
-    table_class: type[DiffMixin],
+    table_class: type[DiffT],
     /,
     *,
     status: Status | list[Status] | None = None,
@@ -435,7 +442,7 @@ def diff_count(
 
 def max_diff_timestamp(
     session: Session,
-    table_class: type[DiffMixin],
+    table_class: type[DiffT],
     /,
     *,
     status: Status | list[Status] | None = None,
@@ -450,7 +457,7 @@ def max_diff_timestamp(
 
 def min_diff_timestamp(
     session: Session,
-    table_class: type[DiffMixin],
+    table_class: type[DiffT],
     /,
     *,
     status: Status | list[Status] | None = None,

tests/integration/database_test.py

@@ -126,31 +126,6 @@ def test_add_revision(db_session):
     assert res == expected
 
 
-@pytest.mark.parametrize(
-    "diffs_data",
-    [
-        {
-            "project": "wikipedia",
-            "lang": "en",
-            "page_namespace": 0,
-            "page_title": "Diff_from_submission_id",
-            "rev_id": 2001,
-            "rev_parent_id": 2000,
-            "rev_timestamp": "20220101010101",
-            "rev_user_text": "Example",
-            "submission_id": str(UUID),
-            "status": database.Status.PENDING.value,
-            "status_timestamp": "20220101020202",
-        },
-    ],
-    indirect=True,
-)
-def test_queued_diff_from_submission_id(db_session, diffs_data):
-    result = database.queued_diff_from_submission_id(db_session, UUID)
-    assert result is not None
-    assert result.submission_id == UUID
-
-
 @pytest.mark.parametrize(
     "diffs_data,status",
     [
@@ -173,20 +148,46 @@ def test_queued_diff_from_submission_id(db_session, diffs_data):
     ],
     indirect=["diffs_data"],
 )
-def test_queued_diffs_by_status(db_session, diffs_data, status):
-    result = database.queued_diffs_by_status(
-        db_session, database.Status(status)
+def test_diffs_by_status(db_session, diffs_data, status):
+    result = database.diffs_by_status(
+        db_session, database.QueuedDiff, database.Status(status)
     )
     assert len(result) == 1
     assert result[0].status is database.Status(status)
-    result = database.queued_diffs_by_status(
+    result = database.diffs_by_status(
         db_session,
+        database.QueuedDiff,
         [database.Status(status)],
         delta=datetime.timedelta(hours=-1),
     )
     assert len(result) == 0
 
 
+@pytest.mark.parametrize(
+    "diffs_data",
+    [
+        {
+            "project": "wikipedia",
+            "lang": "en",
+            "page_namespace": 0,
+            "page_title": "Diff_from_submission_id",
+            "rev_id": 2001,
+            "rev_parent_id": 2000,
+            "rev_timestamp": "20220101010101",
+            "rev_user_text": "Example",
+            "submission_id": str(UUID),
+            "status": database.Status.PENDING.value,
+            "status_timestamp": "20220101020202",
+        },
+    ],
+    indirect=True,
+)
+def test_queued_diff_from_submission_id(db_session, diffs_data):
+    result = database.queued_diff_from_submission_id(db_session, UUID)
+    assert result is not None
+    assert result.submission_id == UUID
+
+
 @pytest.mark.parametrize(
     "diffs_data",
     [

tests/unit/cli_test.py

@@ -69,6 +69,11 @@
             Namespace(action="reports"),
             id="reports",
         ),
+        pytest.param(
+            ("update-ready-diffs",),
+            Namespace(action="update-ready-diffs"),
+            id="update-ready-diffs",
+        ),
         pytest.param(
             ("setup",),
             Namespace(
@@ -102,6 +107,7 @@ def test_parse_script_args(args, expected):
         ("check-changes", "--limit", "ten"),
         ("check-changes", "--workers", "3"),
         ("reports", "foo"),
+        ("update-ready-diffs", "foo"),
         ("setup", "--foo"),
     ],
 )