Dev r84 concurrent push

.gitignore

@@ -2,6 +2,7 @@
 projects*/
 data/
 mergin_db
+diagnostic_logs
 
 logs
 *.log

README.md

@@ -47,7 +47,7 @@ You are currently browsing repository for Mergin Maps web server and web client.
 - 🌱 **Sharing with collaborators** - Projects can be shared with other team members
 - 🏰 **Permission system** - Decide who can read, write or manage projects
 - 🌈 **Web interface** - Simple user interface to view and manage projects
-- ⚡️ **Fast** - Efficient sync protocol transfering data between clients and server
+- ⚡️ **Fast** - Efficient sync protocol transferring data between clients and server
 - 🧑‍💻 **Developer friendly** - Mergin Maps is open platform. CLI tools and client libraries are available for [Python](https://github.com/MerginMaps/python-api-client) and [C++](https://github.com/MerginMaps/cpp-api-client)
 - :camera: **Sync images** - Supporting sync of photos with common cloud storage using [mergin-media-sync](https://github.com/MerginMaps/media-sync) tool
 - 💽 **Sync with database** - Supporting two-way sync of data with PostGIS using [mergin-db-sync](https://github.com/MerginMaps/db-sync) tool
@@ -74,7 +74,7 @@ Admin users can enter the admin interface available at `/admin` URL which provid
 
 ### Contributing
 
-Contributions are welcomed! You can set up development environment by following a guide in [development.md](./deployment/community/development.md). Before you create your first pull request, we kindly ask you to sign the CLA with your GitHub user name and date [here](LICENSES/CLA-signed-list.md).
+Contributions are welcomed! You can set up development environment by following a guide in [development.md](./development.md). Before you create your first pull request, we kindly ask you to sign the CLA with your GitHub user name and date [here](LICENSES/CLA-signed-list.md).
 
 ## Documentation
 
@@ -93,7 +93,7 @@ If you need support, a custom deployment, extending the service capabilities and
 
 Contributions are welcome!
 
-More information for developers can be found in the dedicated [development](./deployment/community/development.md) page.
+More information for developers can be found in the dedicated [development](./development.md) page.
 
 Client side modules:
 - [Python](https://github.com/MerginMaps/python-api-client) client library + CLI

deployment/community/.env.template

@@ -98,12 +98,10 @@ MAIL_SUPPRESS_SEND=0
 
 #MAIL_BCC=''
 
-#MERGIN_LOGO_URL=  # for link to logo in emails
+MERGIN_LOGO_URL=https://merginmaps.com/MM_logo_HORIZ_COLOR_TRANSPARENT_no_padding.png
 
 #MAIL_DEBUG=MAIL_SUPPRESS_SEND | False
 
-
-
 # data sync
 
 #LOCAL_PROJECTS=os.path.join(config_dir, os.pardir, os.pardir, 'projects')  # for local storage type
@@ -215,5 +213,7 @@ NO_MONKEY_PATCH=False
 
 # Diagnostic logs
 
+DIAGNOSTIC_LOGS_URL=
+
 DIAGNOSTIC_LOGS_DIR=/diagnostic_logs
 

deployment/community/docker-compose.yml

@@ -30,7 +30,6 @@ services:
     volumes:
       - ./projects:/data
       - ./diagnostic_logs:/diagnostic_logs
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env
     depends_on:
@@ -48,8 +47,6 @@ services:
     environment:
       - GEVENT_WORKER=0
       - NO_MONKEY_PATCH=1
-    volumes:
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     depends_on:
       - redis
       - server
@@ -68,7 +65,6 @@ services:
       - NO_MONKEY_PATCH=1
     volumes:
       - ./projects:/data
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     depends_on:
       - redis
       - server

deployment/enterprise/.env.template

@@ -175,11 +175,8 @@ ACCOUNT_EXPIRATION=1
 
 # for links generated in emails
 
-#MERGIN_BASE_URL=http://localhost:5000
-MERGIN_BASE_URL=fixme
-
 #MERGIN_LOGO_URL=  # for link to logo in emails
-MERGIN_LOGO_URL=fixme
+MERGIN_LOGO_URL=https://merginmaps.com/MM_logo_HORIZ_COLOR_TRANSPARENT_no_padding.png
 
 # global workspace related bits - ignored in non-CE versions
 # GLOBAL_WORKSPACE mergin
@@ -228,6 +225,8 @@ VECTOR_TILES_STYLE_URL=https://tiles-ee.merginmaps.com//styles/default.json
 ### Diagnostic logs from Mobile and QGIS Plugin
 DIAGNOSTIC_LOGS_DIR=/diagnostic_logs
 
+DIAGNOSTIC_LOGS_URL=
+
 ### SSO ################################################################################################################
 SSO_ENABLED=False
 

deployment/enterprise/docker-compose.yml

@@ -13,9 +13,10 @@ services:
     volumes:
       - ./data:/data  # map data dir to host
       - ./diagnostic_logs:/diagnostic_logs # diagnostic logs dir
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env
+    environment:
+      - GUNICORN_CMD_ARGS="--limit-request-line 8190"
     depends_on:
       - db
     networks:
@@ -56,8 +57,6 @@ services:
     restart: always
     user: 901:999
     command: ["celery -A application.celery beat --loglevel=info"]
-    volumes:
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env
     depends_on:
@@ -75,7 +74,6 @@ services:
     volumes:
       - ./data:/data  # map data dir to host
       - ./map_data:/overviews
-      - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env
     depends_on:

development.md

@@ -43,7 +43,7 @@ $ yarn install
 $ yarn link:dependencies # link dependencies
 $ yarn build:libs # bild libraries @mergin/lib @mergin/admin-lib @mergin/lib-vue2
 $ yarn dev  # development client web application dev server on port 8080 (package @mergin/app)
-$ yarn dev:admin  # development admin appplication dev server on port 8081 (package @mergin/admin-app)
+$ yarn dev:admin  # development admin application dev server on port 8081 (package @mergin/admin-app)
 ```
 
 If you are developing a library package (named **-lib*), it is useful to watch the library for changes instead of rebuilding it each time.
@@ -71,8 +71,6 @@ cd deployment/community/
 # Create .prod.env file from .env.template
 cp .env.template .prod.env
 
-# Run the docker composition with the current Dockerfiles
-cp .env.template .prod.env
 docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
 
 # Give ownership of the ./projects folder to user that is running the gunicorn container
@@ -98,7 +96,7 @@ docker exec -it merginmaps-server flask server send-check-email --email  admin@e
 In docker-compose.dev.yml is started maildev/maildev image that can be used to test emails (see [https://github.com/maildev/maildev/](https://github.com/maildev/maildev/)). In localhost:1080 you can see the emails sent by the application in web interface.
 
 ### Running with remote debugger
-If you want to run the application with remote debugger, you can use debug compose file with attatched source code and reload.
+If you want to run the application with remote debugger, you can use debug compose file with attached source code and reload.
 It starts a debugpy session on port 5678 you can attach to.
 
 ```shell

server/application.py

@@ -27,6 +27,7 @@
     remove_projects_archives,
     remove_temp_files,
     remove_projects_backups,
+    remove_unused_chunks,
 )
 from mergin.celery import celery, configure_celery
 from mergin.stats.config import Configuration
@@ -47,6 +48,7 @@
         "GLOBAL_WRITE",
         "ENABLE_SUPERADMIN_ASSIGNMENT",
         "DIAGNOSTIC_LOGS_URL",
+        "V2_PUSH_ENABLED",
     ]
 )
 register_stats(application)
@@ -85,4 +87,9 @@ def setup_periodic_tasks(sender, **kwargs):
         crontab(hour=3, minute=0),
         remove_projects_archives,
         name="remove old project archives",
+    ),
+    sender.add_periodic_task(
+        crontab(hour="*/4", minute=0),
+        remove_unused_chunks,
+        name="clean up of outdated chunks",
     )

server/mergin/app.py

@@ -12,7 +12,17 @@
 from sqlalchemy.schema import MetaData
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
-from flask import json, jsonify, request, abort, current_app, Flask, Request, Response
+from flask import (
+    json,
+    jsonify,
+    make_response,
+    request,
+    abort,
+    current_app,
+    Flask,
+    Request,
+    Response,
+)
 from flask_login import current_user, LoginManager
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
@@ -25,7 +35,7 @@
 import time
 import traceback
 from werkzeug.exceptions import HTTPException
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Tuple
 
 from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
 from .config import Configuration
@@ -347,6 +357,16 @@ def ping():  # pylint: disable=W0612
         )
         return status, 200
 
+    # reading raw input stream not supported in connexion so far
+    # https://github.com/zalando/connexion/issues/592
+    # and as workaround we use custom Flask endpoint in create_app function
+    @app.route("/v2/projects/<id>/chunks", methods=["POST"])
+    @auth_required
+    def upload_chunk_v2(id: str):
+        from .sync import public_api_v2_controller
+
+        return public_api_v2_controller.upload_chunk(id)
+
     # reading raw input stream not supported in connexion so far
     # https://github.com/zalando/connexion/issues/592
     # and as workaround we use custom Flask endpoint in create_app function
@@ -485,6 +505,12 @@ class ResponseError:
     def to_dict(self) -> Dict:
         return dict(code=self.code, detail=self.detail + f" ({self.code})")
 
+    def response(self, status_code: int) -> Tuple[Response, int]:
+        """Returns a custom error response with the given code."""
+        response = make_response(jsonify(self.to_dict()), status_code)
+        response.headers["Content-Type"] = "application/problem+json"
+        return response, status_code
+
 
 def whitespace_filter(obj):
     return obj.strip() if isinstance(obj, str) else obj

server/mergin/auth/models.py

@@ -36,6 +36,8 @@ class User(db.Model):
         default=datetime.datetime.utcnow,
     )
 
+    last_signed_in = db.Column(db.DateTime(), nullable=True)
+
     __table_args__ = (
         db.Index("ix_user_username", func.lower(username), unique=True),
         db.Index("ix_user_email", func.lower(email), unique=True),
@@ -289,6 +291,7 @@ def __init__(self, user_id: int, ua: str, ip: str, device_id: Optional[str] = No
         self.user_agent = ua
         self.ip_address = ip
         self.device_id = device_id
+        self.timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
 
     @staticmethod
     def add_record(user_id: int, req: request) -> None:
@@ -300,4 +303,37 @@ def add_record(user_id: int, req: request) -> None:
             return
         lh = LoginHistory(user_id, ua, ip, device_id)
         db.session.add(lh)
+
+        # cache user last login
+        User.query.filter_by(id=user_id).update({"last_signed_in": lh.timestamp})
+        db.session.commit()
+
+    @staticmethod
+    def get_users_last_signed_in(user_ids: list) -> dict:
+        """Get users last signed in dates.
+        Result is also cached in User table for future use.
+        """
+        result = (
+            db.session.query(
+                LoginHistory.user_id,
+                func.max(LoginHistory.timestamp).label("last_signed_in"),
+            )
+            .filter(LoginHistory.user_id.in_(user_ids))
+            .group_by(LoginHistory.user_id)
+            .all()
+        )
+
+        user_mapping = [
+            {
+                "id": row.user_id,  # user_id as PK in User table
+                "last_signed_in": row.last_signed_in,
+            }
+            for row in result
+        ]
+        if not user_mapping:
+            return {}
+
+        # cache users last signed in
+        db.session.bulk_update_mappings(User, user_mapping)
         db.session.commit()
+        return {item["id"]: item["last_signed_in"] for item in user_mapping}

server/mergin/sync/commands.py

@@ -11,7 +11,6 @@
 from flask import Flask, current_app
 from sqlalchemy import func
 
-from .files import UploadChanges
 from ..app import db
 from .models import Project, ProjectVersion
 from .utils import split_project_path
@@ -55,8 +54,8 @@ def create(name, namespace, username):  # pylint: disable=W0612
         p = Project(**project_params)
         p.updated = datetime.utcnow()
         db.session.add(p)
-        changes = UploadChanges(added=[], updated=[], removed=[])
-        pv = ProjectVersion(p, 0, user.id, changes, "127.0.0.1")
+        pv = ProjectVersion(p, 0, user.id, [], "127.0.0.1")
+        pv.project = p
         db.session.add(pv)
         db.session.commit()
         os.makedirs(p.storage.project_dir, exist_ok=True)

server/mergin/sync/config.py

@@ -64,3 +64,14 @@ class Configuration(object):
     )
     # in seconds, older unfinished zips are moved to temp
     PARTIAL_ZIP_EXPIRATION = config("PARTIAL_ZIP_EXPIRATION", default=600, cast=int)
+    # whether new push is allowed
+    V2_PUSH_ENABLED = config("V2_PUSH_ENABLED", default=True, cast=bool)
+    # directory for file chunks
+    UPLOAD_CHUNKS_DIR = config(
+        "UPLOAD_CHUNKS_DIR",
+        default=os.path.join(LOCAL_PROJECTS, "chunks"),
+    )
+    # time in seconds after chunks are permanently deleted (1 day)
+    UPLOAD_CHUNKS_EXPIRATION = config(
+        "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
+    )

server/mergin/sync/db_events.py

@@ -6,6 +6,8 @@
 from flask import current_app, abort
 from sqlalchemy import event
 
+from .models import ProjectVersion
+from .tasks import optimize_storage
 from ..app import db
 
 
@@ -14,9 +16,17 @@ def check(session):
         abort(503, "Service unavailable due to maintenance, please try later")
 
 
+def optimize_gpgk_storage(mapper, connection, project_version):
+    # do not optimize on every version, every 10th is just fine
+    if not project_version.name % 10:
+        optimize_storage.delay(project_version.project_id)
+
+
 def register_events():
     event.listen(db.session, "before_commit", check)
+    event.listen(ProjectVersion, "after_insert", optimize_gpgk_storage)
 
 
 def remove_events():
     event.remove(db.session, "before_commit", check)
+    event.listen(ProjectVersion, "after_insert", optimize_gpgk_storage)