Merge branch 'master' into release

.env.example

@@ -67,6 +67,10 @@ SENTRY_DSN=
 # DEFAULT: 1
 SENTRY_SAMPLE_RATE=1
 
+# Release version shown on sentry. Read more on https://docs.sentry.io/product/releases/.
+# DEFAULT: dev
+SENTRY_RELEASE=dev
+
 REDIS_PASSWORD=change_me_with_a_very_loooooooooooong_password
 REDIS_PORT=6379
 
@@ -143,6 +147,14 @@ COMPOSE_FILE=docker-compose.yml:docker-compose.override.local.yml
 # DEFAULT: :
 COMPOSE_PATH_SEPARATOR=:
 
+# Debugpy port used for the `app` service
+# DEFAULT: 5678
+DEBUG_DEBUGPY_APP_PORT=5678
+
+# Debugpy port used for the `worker_wrapper` service
+# DEFAULT: 5679
+DEBUG_DEBUGPY_WORKER_WRAPPER_PORT=5679
+
 # Path to the nginx, letsencrypt, etc configuration files, used by script in `./scripts/`.
 # DEFAULT: ./conf
 CONFIG_PATH=./conf

.gitignore

@@ -11,3 +11,4 @@ client/projects
 conf/nginx/certs/*
 conf/certbot/*
 Pipfile*
+**/site-packages

README.md

@@ -77,22 +77,10 @@ To run only a test module (e.g. `test_permission.py`)
 
 > This section gives examples for VSCode, please adapt to your IDE)
 
-If using the provided docker-compose overrides for developement, `debugpy` is installed.
+If you are using the provided `docker-compose.override.local.yml`, then `debugpy` is automatically installed and configured for use.
 
-You can debug interactively by adding this snipped anywhere in the code.
-```python
-import debugpy
-debugpy.listen(("0.0.0.0", 5680))
-print("debugpy waiting for debugger... 🐛")
-debugpy.wait_for_client()  # optional
-```
-
-Or alternativley, prefix your commands with `python -m debugpy --listen 0.0.0.0:5680 --wait-for-client`.
+Add the following to your IDE to connect (example given for VSCode's `.vscode/launch.json`, triggered with `F5`):
 
-    docker compose run app -p 5680:5680 python -m debugpy --listen 0.0.0.0:5680 --wait-for-client manage.py test
-    docker compose run worker_wrapper -p 5681:5681 python -m debugpy --listen 0.0.0.0:5681 --wait-for-client manage.py test
-
-Then, configure your IDE to connect (example given for VSCode's `.vscode/launch.json`, triggered with `F5`):
 ```
 {
     "version": "0.2.0",
@@ -102,27 +90,65 @@ Then, configure your IDE to connect (example given for VSCode's `.vscode/launch.
             "type": "python",
             "request": "attach",
             "justMyCode": false,
-            "connect": {"host": "localhost", "port": 5680},
-            "pathMappings": [{
-                "localRoot": "${workspaceFolder}/docker-app/qfieldcloud",
-                "remoteRoot": "/usr/src/app/qfieldcloud"
-            }]
+            "connect": {"host": "localhost", "port": 5678},
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/docker-app/qfieldcloud",
+                    "remoteRoot": "/usr/src/app/qfieldcloud"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/docker-app/site-packages",
+                    "remoteRoot": "/usr/local/lib/python3.10/site-packages/"
+                },
+            ],
         },
         {
             "name": "QFC debug worker_wrapper",
             "type": "python",
             "request": "attach",
             "justMyCode": false,
-            "connect": {"host": "localhost", "port": 5681},
-            "pathMappings": [{
-                "localRoot": "${workspaceFolder}/docker-app/qfieldcloud",
-                "remoteRoot": "/usr/src/app/qfieldcloud"
-            }]
+            "connect": {"host": "localhost", "port": 5679},
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/docker-app/qfieldcloud",
+                    "remoteRoot": "/usr/src/app/qfieldcloud"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/docker-app/site-packages",
+                    "remoteRoot": "/usr/local/lib/python3.10/site-packages/"
+                },
+            ],
         }
     ]
 }
 ```
 
+To add breakpoints in vendor modules installed via `pip` or `apt`, you need a copy of their source code.
+The easiest way to achieve that is do actual copy of them:
+
+```
+docker compose cp app:/usr/local/lib/python3.10/site-packages/ docker-app/site-packages
+```
+
+The configuration for the vendor modules is the second object in the example `pathMappings` above, as well as setting `justMyCode` to `false`.
+
+Do not forget to copy the site packages every time any of the `requirements.txt` files are changed!
+
+If you are not using `docker-compose.override.local.yml`, there are other options.
+
+You can debug interactively by adding this snippet anywhere in the code.
+```python
+import debugpy
+debugpy.listen(("0.0.0.0", 5680))
+print("debugpy waiting for debugger... 🐛")
+debugpy.wait_for_client()  # optional
+```
+
+Or alternativley, prefix your commands with `python -m debugpy --listen 0.0.0.0:5680 --wait-for-client`.
+
+    docker compose run app -p 5680:5680 python -m debugpy --listen 0.0.0.0:5680 --wait-for-client manage.py test
+    docker compose run worker_wrapper -p 5681:5681 python -m debugpy --listen 0.0.0.0:5681 --wait-for-client manage.py test
+
 Note if you run tests using the `docker-compose.test.yml` configuration, the `app` and `worker-wrapper` containers expose ports `5680` and `5681` respectively.
 
 

docker-app/qfieldcloud/core/admin.py

@@ -71,6 +71,9 @@ class ModelAdminEstimateCountMixin:
     # Display '(Show all)' instead of '(<count>)' in search bar
     show_full_result_count = False
 
+    # Overwrite the default Django configuration of 100
+    list_per_page = settings.QFIELDCLOUD_ADMIN_LIST_PER_PAGE
+
 
 class QFieldCloudModelAdmin(ModelAdminEstimateCountMixin, admin.ModelAdmin):
     pass
@@ -390,7 +393,7 @@ class PersonAdmin(QFieldCloudModelAdmin):
         "is_active",
         "date_joined",
         "last_login",
-        "storage_usage__field",
+        # "storage_usage__field",
     )
     list_filter = (
         "type",
@@ -402,6 +405,7 @@ class PersonAdmin(QFieldCloudModelAdmin):
     search_fields = ("username__icontains", "email__iexact")
 
     fields = (
+        "storage_usage__field",
         "username",
         "password",
         "email",
@@ -547,6 +551,8 @@ class ProjectAdmin(QFieldCloudModelAdmin):
         "description",
         "is_public",
         "owner",
+        "status",
+        "status_code",
         "project_filename",
         "file_storage_bytes",
         "created_at",
@@ -558,6 +564,8 @@ class ProjectAdmin(QFieldCloudModelAdmin):
     )
     readonly_fields = (
         "id",
+        "status",
+        "status_code",
         "file_storage_bytes",
         "created_at",
         "updated_at",
@@ -1005,6 +1013,7 @@ class OrganizationAdmin(QFieldCloudModelAdmin):
         TeamInline,
     )
     fields = (
+        "storage_usage__field",
         "username",
         "email",
         "organization_owner",
@@ -1015,7 +1024,7 @@ class OrganizationAdmin(QFieldCloudModelAdmin):
         "email",
         "organization_owner__link",
         "date_joined",
-        "storage_usage__field",
+        # "storage_usage__field",
         "active_users",
     )
 

docker-app/qfieldcloud/core/migrations/0064_auto_20230328_2017.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.18 on 2023-03-28 18:17
+
+import migrate_sql.operations
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0063_auto_20230305_1551"),
+    ]
+
+    operations = [
+        migrate_sql.operations.ReverseAlterSQL(
+            name="core_user_email_partial_uniq",
+            sql="\n            DROP INDEX IF EXISTS core_user_email_partial_uniq\n        ",
+            reverse_sql="\n            CREATE UNIQUE INDEX IF NOT EXISTS core_user_email_partial_uniq ON core_user (email)\n            WHERE type = 1 AND email IS NOT NULL AND email != ''\n        ",
+        ),
+        migrate_sql.operations.AlterSQL(
+            name="core_user_email_partial_uniq",
+            sql="\n            CREATE UNIQUE INDEX IF NOT EXISTS core_user_email_partial_uniq ON core_user (UPPER(email))\n            WHERE type = 1 AND email IS NOT NULL AND email != ''\n        ",
+            reverse_sql="\n            DROP INDEX IF EXISTS core_user_email_partial_uniq\n        ",
+        ),
+    ]

docker-app/qfieldcloud/core/sql_config.py

@@ -235,7 +235,7 @@
     SQLItem(
         "core_user_email_partial_uniq",
         r"""
-            CREATE UNIQUE INDEX IF NOT EXISTS core_user_email_partial_uniq ON core_user (email)
+            CREATE UNIQUE INDEX IF NOT EXISTS core_user_email_partial_uniq ON core_user (UPPER(email))
             WHERE type = 1 AND email IS NOT NULL AND email != ''
         """,
         r"""

docker-app/qfieldcloud/settings.py

@@ -415,6 +415,9 @@ def before_send(event, hint):
     "/admin/account/emailaddress/admin/export_emails_to_csv/",
 )
 
+# Sets the default admin list view per page, the Django default is 100
+QFIELDCLOUD_ADMIN_LIST_PER_PAGE = 20
+
 # Use pg meta table estimate for pagination and display above n entries
 QFIELDCLOUD_ADMIN_EXACT_COUNT_LIMIT = 10000
 

docker-compose.override.local.yml

@@ -9,12 +9,13 @@ services:
     ports:
       # allow direct access without nginx
       - ${DJANGO_DEV_PORT}:8000
+      - ${DEBUG_DEBUGPY_APP_PORT:-5678}:5678
     volumes:
       # mount the source for live reload
       - ./docker-app/qfieldcloud:/usr/src/app/qfieldcloud
     environment:
       DEBUG: 1
-    command: python3 manage.py runserver 0.0.0.0:8000
+    command: python3 -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000
     depends_on:
       - db
       - redis
@@ -24,10 +25,13 @@ services:
     build:
       args:
         - DEBUG_BUILD=1
+    ports:
+      - ${DEBUG_DEBUGPY_WORKER_WRAPPER_PORT:-5679}:5679
     volumes:
       # mount the source for live reload
       - ./docker-app/qfieldcloud:/usr/src/app/qfieldcloud
       - ./docker-app/worker_wrapper:/usr/src/app/worker_wrapper
+    command: python3 -m debugpy --listen 0.0.0.0:5679 manage.py dequeue
 
   smtp4dev:
     image: rnwood/smtp4dev:v3

docker-compose.override.test.yml

@@ -12,18 +12,12 @@ services:
       # we must use the same db for test and runserver
       SQL_DATABASE: test_${POSTGRES_DB}
       SQL_DATABASE_TEST: test_${POSTGRES_DB}
-    ports:
-      - 5680:5680
-    command: python3 -m debugpy --listen 0.0.0.0:5680 manage.py runserver 0.0.0.0:8000
 
   worker_wrapper:
     environment:
       # we must use the same db for test and runserver
       SQL_DATABASE: test_${POSTGRES_DB}
       SQL_DATABASE_TEST: test_${POSTGRES_DB}
-    command: python3 -m debugpy --listen 0.0.0.0:5681 manage.py dequeue
-    ports:
-      - 5681:5681
     scale: ${QFIELDCLOUD_WORKER_REPLICAS}
 
   db:

docker-compose.yml

@@ -32,6 +32,7 @@ services:
       DEBUG: ${DEBUG}
       ENVIRONMENT: ${ENVIRONMENT}
       SENTRY_DSN: ${SENTRY_DSN}
+      SENTRY_RELEASE: ${SENTRY_RELEASE}
       # Sentry environment should not be configured like this, but I never made it work with `sentry_sdk.init(environment=ENVIRONMENT)`.
       SENTRY_ENVIRONMENT: ${ENVIRONMENT}
       SENTRY_SAMPLE_RATE: ${SENTRY_SAMPLE_RATE}

docker-qgis/utils.py

@@ -584,6 +584,7 @@ def get_layers_data(project: QgsProject) -> Dict[str, Dict]:
             "error_summary": error.summary() if error.messageList() else "",
             "error_message": layer.error().message(),
             "filename": layer_source.filename,
+            "provider_name": None,
             "provider_error_summary": None,
             "provider_error_message": None,
         }
@@ -611,6 +612,8 @@ def get_layers_data(project: QgsProject) -> Dict[str, Dict]:
                 "provider_error_message"
             ] = data_provider_error.message()
 
+            layers_by_id[layer_id]["provider_name"] = data_provider.name()
+
             if not layers_by_id[layer_id]["provider_error_summary"]:
                 service = data_provider.uri().service()
                 if service:

scripts/check_envvars.sh

@@ -1,3 +1,3 @@
 #!/bin/bash -e
 
-python3 scripts/check_envvars.py .env.example --docker-compose-dir . --ignored-varnames CONFIG_PATH
+python3 scripts/check_envvars.py .env.example --docker-compose-dir . --ignored-varnames CONFIG_PATH DEBUG_DEBUGPY_APP_PORT DEBUG_DEBUGPY_WORKER_WRAPPER_PORT