Dev r85 v2 pull

server/.test.env

@@ -24,3 +24,4 @@ SECURITY_BEARER_SALT='bearer'
 SECURITY_EMAIL_SALT='email'
 SECURITY_PASSWORD_SALT='password'
 DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs
+GEVENT_WORKER=0

server/mergin/sync/commands.py

@@ -127,3 +127,50 @@ def remove(project_name):
         project.removed_by = None
         db.session.commit()
         click.secho("Project removed", fg="green")
+
+    @project.command()
+    @click.argument("project-name", callback=normalize_input(lowercase=False))
+    @click.option("--since", type=int, required=False)
+    @click.option("--to", type=int, required=False)
+    def create_checkpoint(project_name, since=None, to=None):
+        """Create project delta checkpoint, corresponding lower checkpoints and merged diffs for project"""
+        ws, name = split_project_path(project_name)
+        workspace = current_app.ws_handler.get_by_name(ws)
+        if not workspace:
+            click.secho("ERROR: Workspace does not exist", fg="red", err=True)
+            sys.exit(1)
+        project = (
+            Project.query.filter_by(workspace_id=workspace.id, name=name)
+            .filter(Project.storage_params.isnot(None))
+            .first()
+        )
+        if not project:
+            click.secho("ERROR: Project does not exist", fg="red", err=True)
+            sys.exit(1)
+
+        since = since if since is not None else 0
+        to = to if to is not None else project.latest_version
+        if since < 0 or to < 1:
+            click.secho(
+                "ERROR: Invalid version number, minimum version for 'since' is 0 and minimum version for 'to' is 1",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if to > project.latest_version:
+            click.secho(
+                "ERROR: 'to' version exceeds latest project version", fg="red", err=True
+            )
+            sys.exit(1)
+
+        if since >= to:
+            click.secho(
+                "ERROR: 'since' version must be less than 'to' version",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        project.get_delta_changes(since, to)
+        click.secho("Project checkpoint(s) created", fg="green")

server/mergin/sync/config.py

@@ -75,3 +75,5 @@ class Configuration(object):
     UPLOAD_CHUNKS_EXPIRATION = config(
         "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
     )
+    # whether client can pull using v2 apis
+    V2_PULL_ENABLED = config("V2_PULL_ENABLED", default=True, cast=bool)

server/mergin/sync/errors.py

@@ -95,3 +95,10 @@ def to_dict(self) -> Dict:
 class BigChunkError(ResponseError):
     code = "BigChunkError"
     detail = f"Chunk size exceeds maximum allowed size {MAX_CHUNK_SIZE} MB"
+
+
+class DiffDownloadError(ResponseError):
+    code = "DiffDownloadError"
+    detail = (
+        "Required diff file could not be downloaded as it could not be reconstructed"
+    )

server/mergin/sync/files.py

@@ -4,11 +4,18 @@
 import datetime
 from enum import Enum
 import os
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, List
 import uuid
 from flask import current_app
-from marshmallow import ValidationError, fields, EXCLUDE, post_dump, validates_schema
+from marshmallow import (
+    ValidationError,
+    fields,
+    EXCLUDE,
+    post_dump,
+    validates_schema,
+    post_load,
+)
 from pathvalidate import sanitize_filename
 
 from .utils import (
@@ -231,11 +238,124 @@ def validate(self, data, **kwargs):
 
 class ProjectFileSchema(FileSchema):
     mtime = DateTimeWithZ()
-    diff = fields.Nested(FileSchema())
+    diff = fields.Nested(
+        FileSchema(),
+    )
 
     @post_dump
     def patch_field(self, data, **kwargs):
         # drop 'diff' key entirely if empty or None as clients would expect
         if not data.get("diff"):
             data.pop("diff", None)
         return data
+
+
+@dataclass
+class DeltaDiffFile:
+    """Diff file path in diffs list"""
+
+    path: str
+
+
+class DeltaChangeDiffFileSchema(ma.Schema):
+    """Schema for diff file path in diffs list"""
+
+    path = fields.String(required=True)
+
+
+@dataclass
+class DeltaChangeBase(File):
+    """Base class for changes stored in json list or returned from delta endpoint"""
+
+    change: PushChangeType
+    version: int
+
+
+@dataclass
+class DeltaChangeMerged(DeltaChangeBase):
+    """Delta item with merged diffs to list of multiple diff files"""
+
+    diffs: List[DeltaDiffFile] = field(default_factory=list)
+
+    def to_data_delta(self):
+        """Convert DeltaMerged to DeltaData with single diff"""
+        result = DeltaChange(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diffs:
+            result.diff = self.diffs[0].path
+        return result
+
+
+@dataclass
+class DeltaChange(DeltaChangeBase):
+    """Change items stored in database as list of this item with single diff file"""
+
+    diff: Optional[str] = None
+
+    def to_merged(self) -> DeltaChangeMerged:
+        """Convert to DeltaMerged with multiple diffs"""
+        result = DeltaChangeMerged(
+            path=self.path,
+            size=self.size,
+            checksum=self.checksum,
+            change=self.change,
+            version=self.version,
+        )
+        if self.diff:
+            result.diffs = [DeltaDiffFile(path=self.diff)]
+        return result
+
+
+class DeltaChangeBaseSchema(ma.Schema):
+    """Base schema for delta json and response from delta endpoint"""
+
+    path = fields.String(required=True)
+    size = fields.Integer(required=True)
+    checksum = fields.String(required=True)
+    version = fields.Integer(required=True)
+    change = fields.Enum(PushChangeType, by_value=True, required=True)
+
+
+class DeltaChangeSchema(DeltaChangeBaseSchema):
+    """Schema for change data in changes column"""
+
+    diff = fields.String(required=False)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return DeltaChange(**data)
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diff' key entirely if empty or None as database would expect
+        if not data.get("diff"):
+            data.pop("diff", None)
+        return data
+
+
+class DeltaChangeItemSchema(DeltaChangeBaseSchema):
+    """Schema for delta changes response"""
+
+    version = fields.Function(lambda obj: f"v{obj.version}")
+    diffs = fields.List(fields.Nested(DeltaChangeDiffFileSchema()))
+
+    @post_dump
+    def patch_field(self, data, **kwargs):
+        # drop 'diffs' key entirely if empty or None as clients would expect
+        if not data.get("diffs"):
+            data.pop("diffs", None)
+        return data
+
+
+class DeltaChangeRespSchema(ma.Schema):
+    """Schema for list of delta changes wrapped in items field"""
+
+    items = fields.List(fields.Nested(DeltaChangeItemSchema()))
+
+    class Meta:
+        unknown = EXCLUDE