feat(copernicus): Collect weather data from Copernicus API (#3)

* feat(copernicus): Collect weather data from Copernicus API

* Run lint

* Rolling back __init__ to older version

* Minor improvements and delete NetCDF after inserting

* Code usage

* Add copernicus task to celery beat

* Minor deps fix

* Move notebooks to its own dir

* Change to beat inicialization

* Ability of retrieving raw values - 3h intervals

* Raw values

* Add cache to dataframe

* Difference between InfoDengue and Copernicus data

* Fix task initialization

* Crontab is not working for scheduling a task

* Send task at startup

.dockerignore

@@ -0,0 +1 @@
+*/data/*.nc

.gitignore

@@ -88,3 +88,9 @@ downloads.db
 *docker/dbschemas/
 */docker/dev_dumps/
 *.gz
+
+data/*
+
+celerybeat-schedule.bak
+celerybeat-schedule.dat
+celerybeat-schedule.dir

conda/dev.yaml

@@ -3,16 +3,9 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - docker-compose #
-  - python-dotenv #
-  - celery #
-  - earthengine-api
-  - geoviews
-  - imageio
+  - docker-compose
+  - celery
   - make
   - poetry
-  - pydrive
   - python=3.9
-  - rasterio
-  - rasterstats
-  - sqlalchemy #
+

docker/celery/run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-exec celery -A satellite_weather_downloader.celeryapp worker -l INFO
+exec celery --workdir /opt/services/satellite_weather_downloader/celery_app --config celeryconfig -A tasks worker -B --loglevel=INFO

docker/compose-base.yaml

@@ -32,11 +32,12 @@ services:
       - satellite
     volumes:
       - ../../logs:/var/log/
+      - ../satellite_weather_downloader/celery_app:/opt/services/satellite_weather_downloader/celery_app
     depends_on:
       - rabbitmq
     restart: always
     healthcheck:
-      test: ["CMD-SHELL", 'celery --app satellite_weather_downloader.celeryapp.app inspect ping -d "celery@$${HOSTNAME}"']
+      test: ["CMD-SHELL", 'celery --app satellite_weather_downloader.celery_app.celeryapp.app inspect ping -d "celery@$${HOSTNAME}"']
       interval: 60s
       timeout: 10s
       retries: 5