Change way celery is working

.travis.yml

@@ -2,73 +2,73 @@ dist: bionic
 sudo: required
 language: python
 python:
-- '3.6'
-- '3.7'
-- '3.8'
+  - '3.6'
+  - '3.7'
+  - '3.8'
 
 env:
   matrix:
-  - DJANGO_VERSION=2.2.*
-  - DJANGO_VERSION=3.0.*
-  - DJANGO_VERSION=dev
+    - DJANGO_VERSION=2.2.*
+    - DJANGO_VERSION=3.0.*
+    - DJANGO_VERSION=dev
 
 services:
-- postgresql
+  - postgresql
 
 addons:
   postgresql: '10'
   apt:
     packages:
-    - postgresql-10-postgis-2.4
-    - postgresql-10-pgrouting
+      - postgresql-10-postgis-2.4
+      - postgresql-10-pgrouting
 
 stages:
-- lint
-- test
-- deploy
+  - lint
+  - test
+  - deploy
 
 install:
-- sudo apt-get -y install libproj-dev binutils gdal-bin libgdal-dev
-- pip install -e .[dev]
-- pip install codecov
-- if [[ $DJANGO_VERSION == dev ]]; then pip install -e git+https://github.com/django/django@master#egg=django;
-  else pip install Django==$DJANGO_VERSION -U; fi
+  - sudo apt-get -y install libproj-dev binutils gdal-bin libgdal-dev
+  - pip install -e .[dev]
+  - pip install codecov
+  - if [[ $DJANGO_VERSION == dev ]]; then pip install -e git+https://github.com/django/django@master#egg=django;
+    else pip install Django==$DJANGO_VERSION -U; fi
 
 before_script:
-- psql -c "CREATE USER travis_ci_test WITH ENCRYPTED PASSWORD 'travis_ci_test';" -U
-  postgres
-- psql -c "ALTER USER travis_ci_test WITH SUPERUSER;" -U postgres
-- psql -c 'CREATE DATABASE travis_ci_test WITH OWNER travis_ci_test;' -U postgres
-- psql -d travis_ci_test -c 'CREATE EXTENSION postgis;'
-- psql -d travis_ci_test -c 'CREATE EXTENSION pgrouting;'
+  - psql -c "CREATE USER travis_ci_test WITH ENCRYPTED PASSWORD 'travis_ci_test';" -U
+    postgres
+  - psql -c "ALTER USER travis_ci_test WITH SUPERUSER;" -U postgres
+  - psql -c 'CREATE DATABASE travis_ci_test WITH OWNER travis_ci_test;' -U postgres
+  - psql -d travis_ci_test -c 'CREATE EXTENSION postgis;'
+  - psql -d travis_ci_test -c 'CREATE EXTENSION pgrouting;'
 
 after_failure:
-- pip freeze
+  - pip freeze
 
 script:
-- coverage run ./manage.py test
-- codecov
+  - coverage run ./manage.py test
+  - codecov
 
 jobs:
   allow_failures:
     - env: DJANGO_VERSION=dev
 
   include:
-  - stage: lint
-    install:
-    - pip install black flake8
-    script:
-    - black --check django_geosource
-    - flake8 django_geosource
-  - stage: deploy
-    install: skip
-    before_script: skip
-    script: skip
-    deploy:
-      stage: deploy
-      provider: pypi
-      user: __token__
-      password:
-        secure: jrGnKBj18FS7vcwJ8pmi7/9/EfVzncyjkpzTzoVjRIDbs9N773cpouueJUmD+H6JiLQBI2qaZ0ChcY8fb4MqEqN9MQdjtDFcmvLWkuvTsKq087pYXss0ZLikNP+rbv+xR53KF5yZTTxASUPQkcARqTfk9pCiRJSRLaSuWtQSlrBYt/fo3YvOcGAqoRyMWKRLgJGAabHmOAnWz37ZxSDr36MMfXpLb7FpT9YOS+mEdWog13wSwghUerVKbdjrK/IPtVjpeU3egqYu9P/Z3IoweWkcDEeDR46jxRpw8dU8P9RoFs4RkjQNrXvyKdgxMywQV3sgOiVakqKaXCZAAnshzvcmwt0cEslg5ciYqK2fHU2sXYHPfamThbyrj399WNwgK8B6EwM94YFsr1tAalZpfDWy1vjJ5pA1I2evf3MvlZDHLpoiFk4JKA/13IZIANgVE2c/s2zZL0gkwU8ZY0nROpVh3NwVLLs/QAtXnayKd31R0+5dczJKMgxdbvR2aRNAJ9f2VRdNx/mtJH1QW6M98R89Nt2CrFos2aPfay/C9pOyFIFBpM2OdiJsM1wMSRCmWh21+NhSQUiLzZP++g5uX7V7nseHA0Z51Vbf4AHHUXeDR2WHdJlklFtk1VnSBPex2sWrKHQPLDakrgtlbkzlT66+Lh2NZZ8qIh3XwwUXFF8=
-      on:
-        tags: true
+    - stage: lint
+      install:
+        - pip install black flake8
+      script:
+        - black --check django_geosource
+        - flake8 django_geosource
+    - stage: deploy
+      install: skip
+      before_script: skip
+      script: skip
+      deploy:
+        stage: deploy
+        provider: pypi
+        user: __token__
+        password:
+          secure: jrGnKBj18FS7vcwJ8pmi7/9/EfVzncyjkpzTzoVjRIDbs9N773cpouueJUmD+H6JiLQBI2qaZ0ChcY8fb4MqEqN9MQdjtDFcmvLWkuvTsKq087pYXss0ZLikNP+rbv+xR53KF5yZTTxASUPQkcARqTfk9pCiRJSRLaSuWtQSlrBYt/fo3YvOcGAqoRyMWKRLgJGAabHmOAnWz37ZxSDr36MMfXpLb7FpT9YOS+mEdWog13wSwghUerVKbdjrK/IPtVjpeU3egqYu9P/Z3IoweWkcDEeDR46jxRpw8dU8P9RoFs4RkjQNrXvyKdgxMywQV3sgOiVakqKaXCZAAnshzvcmwt0cEslg5ciYqK2fHU2sXYHPfamThbyrj399WNwgK8B6EwM94YFsr1tAalZpfDWy1vjJ5pA1I2evf3MvlZDHLpoiFk4JKA/13IZIANgVE2c/s2zZL0gkwU8ZY0nROpVh3NwVLLs/QAtXnayKd31R0+5dczJKMgxdbvR2aRNAJ9f2VRdNx/mtJH1QW6M98R89Nt2CrFos2aPfay/C9pOyFIFBpM2OdiJsM1wMSRCmWh21+NhSQUiLzZP++g5uX7V7nseHA0Z51Vbf4AHHUXeDR2WHdJlklFtk1VnSBPex2sWrKHQPLDakrgtlbkzlT66+Lh2NZZ8qIh3XwwUXFF8=
+        on:
+          tags: true

CHANGES.md

@@ -1,49 +1,54 @@
 
+0.4.0 / 2020-03-19
+==================
+
+  * BREAKING CHANGE : change way celery is working to allow using celery in another app
+
 0.3.7 / 2020-03-17
 ==================
 
   * Enhance tests to valide search and filter
   * Add option to sync sources to have more control
   * Add zipfile shapefilesource
 
-0.3.6       (2019-12-19)
-------------------------
+0.3.6 / 2019-12-19
+==================
 
-* Fix bug with FileSourceSerializer
+  * Fix bug with FileSourceSerializer
 
-0.3.5       (2019-12-18)
-------------------------
+0.3.5 / 2019-12-18
+==================
 
-* Add ordering and filtering for sources
-* Add flake8 linting to CI
+  * Add ordering and filtering for sources
+  * Add flake8 linting to CI
 
-0.3.4      (2019-12-16)
------------------------
+0.3.4 / 2019-12-16
+==================
 
 ### Improves
 
-* Improve documentation
-* Fix python3.8, django 3.0 and DRF 3.11 compatibility
+  * Improve documentation
+  * Fix python3.8, django 3.0 and DRF 3.11 compatibility
 
-0.3.3      (2019-11-06)
------------------------
+0.3.3 / 2019-11-06
+==================
 
 ### Improves
 
-* Define MANIFEST.in
+  * Define MANIFEST.in
 
-0.3.1      (2019-11-06)
------------------------
+0.3.1 / 2019-11-06
+==================
 
 ### Improves
 
-* Improve error message when identifier field is not found in the source
-* Improve error message when geojson features has bad geometries
-* Use black for linting in pipelines
+  * Improve error message when identifier field is not found in the source
+  * Improve error message when geojson features has bad geometries
+  * Use black for linting in pipelines
 
-0.3.0      (2019-10-18)
------------------------
+0.3.0 / 2019-10-18
+==================
 
 ### Release
 
-* First release
+  * First release

README.md

@@ -2,7 +2,7 @@
 [![codecov](https://codecov.io/gh/Terralego/django-geosource/branch/master/graph/badge.svg)](https://codecov.io/gh/Terralego/django-geosource)
 [![PyPi version](https://pypip.in/v/django-geosource/badge.png)](https://pypi.org/project/django-geosource/)
 
-# Django GEOSource
+# Django GeoSource
 
 This django application provide a Rest Framework API that allow to manage many geo data
 sources and integrate that data wherever you need, like a Django model or any output
@@ -16,57 +16,66 @@ time. It allow to prevent when a task is stuck and disallow launching one more.
 ## Configure and run Celery
 
 You must define in your project settings the variables CELERY_BROKER_URL and CELERY_RESULT_BACKEND as specified in Celery documentation.
+You also need to create the celery app following this [documentation](https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html#django-first-steps).
 
-To run the celery worker:
+Then to run the celery worker:
 `$ celery worker -A django_geosource -l info`
 
 To run the celery beat worker that allow to synchronize periodically sources, launch this command:
 `$ celery beat --scheduler django_geosource.celery.schedulers.GeosourceScheduler -A django_geosource -l info`
 
 ## Configure data destination
+
 Now, you must set the callback methods that are used to insert data in your destination database.
 
 If you use django-geostore, we provide a set of callback in the `geostore_callbacks` module, else you can define your
 own callbacks.
 
 ### GEOSOURCE_LAYER_CALLBACK
+
 The callback signature receive as first argument the SourceModel object, and must return your Layer object.
 Example:
+
 ```python
 def layer_callback(geosource):
     return Layer.objects.get_or_create(name=geosource.name)[0]
 ```
 
 ### GEOSOURCE_FEATURE_CALLBACK
+
 This one, define a feature creation callback method.
 Example:
+
 ```python
 def feature_callback(geosource, layer, identifier, geometry, attributes):
     return Feature.objects.get_or_create(layer=layer, identifier=identifier, geom=geometry, properties=attributes)[0]
 ```
 
 ### GEOSOURCE_CLEAN_FEATURE_CALLBACK
+
 This callback is called when the refresh is done, to clear old features that are not anymore present in the database.
 It receives as parametter the geosource, layer and begin update date, so you can advise what to do depending of your
 models.
 Example:
+
 ```python
 def clear_features(geosource, layer, begin_date):
     return layer.features.filter(updated_at__lt=begin_date).delete()
 ```
 
 ### GEOSOURCE_DELETE_LAYER_CALLBACK
+
 This is called when a Source is deleted, so you are able to do what you want with the loaded content in database, when
 the source doesn't exist anymore. It's executed before real deletion.
 Example:
+
 ```python
 def delete_layer(geosource, layer):
     if layer.features.count() > 0:
         layer.features.delete()
     return layer.delete()
 ```
 
-
 ## To start a dev instance
 
 Define settings you wants in `test_geosource` django project.
@@ -76,8 +85,7 @@ docker-compose build
 docker-compose up
 ```
 
-First start should failed as the database need to be initialized. Just launch
-the same command twice.
+First start should failed as the database need to be initialized. Just launch the same command twice.
 
 Then initialize the database:
 
@@ -90,7 +98,6 @@ this is an autoreload server.
 
 You can access to the api on http://localhost:8000/api/
 
-
 ## Test and coverage
 
 To run test suite, just launch:

django_geosource/VERSION.md

@@ -1 +1 @@
-0.3.7
+0.4.0

django_geosource/app_settings.py

@@ -1,6 +1,5 @@
 from django.conf import settings
 
-
 # Max time a task can be running until another one can be runned.
 # This is to prevent when a task is blocked.
 MAX_TASK_RUNTIME = getattr(settings, "GEOSOURCE_MAX_TASK_RUNTIME", 24)

django_geosource/geostore_callbacks.py

@@ -4,7 +4,6 @@
 from django.contrib.gis.geos import GEOSGeometry
 from geostore.models import Layer, LayerGroup
 
-
 logger = logging.getLogger(__name__)
 
 

django_geosource/management/commands/resync_all_sources.py

@@ -1,5 +1,4 @@
 from django.core.management import BaseCommand
-
 from django_geosource.models import Source
 
 

django_geosource/management/commands/resync_source.py

@@ -1,5 +1,4 @@
 from django.core.management import BaseCommand
-
 from django_geosource.models import Source
 
 

django_geosource/migrations/0001_initial.py

@@ -2,10 +2,10 @@
 
 import django.contrib.postgres.fields.jsonb
 import django.core.validators
-from django.db import migrations, models
 import django.db.models.deletion
 import django_geosource.mixins
 import django_geosource.models
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

django_geosource/migrations/0004_auto_20190529_1647.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-05-29 14:47
 
-from django.db import migrations, models
 import django_geosource.models
+from django.db import migrations, models
 from django_geosource.models import FieldTypes
 
 

django_geosource/migrations/0007_commandsource.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-06-18 13:00
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

django_geosource/migrations/0010_wmtssource.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-06-28 09:46
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

django_geosource/migrations/0011_auto_20190628_1449.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-06-28 12:49
 
-from django.db import migrations
 import django_geosource.fields
+from django.db import migrations
 
 
 class Migration(migrations.Migration):

django_geosource/migrations/0013_auto_20190709_1735.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-07-09 15:35
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

django_geosource/migrations/0014_auto_20190821_1648.py

@@ -1,7 +1,7 @@
 # Generated by Django 2.0.13 on 2019-08-21 14:48
 
-from django.db import migrations, models
 import django.utils.timezone
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):