diff --git a/satellite_weather_downloader/celery_app/tasks.py b/satellite_weather_downloader/celery_app/tasks.py index af280db..d594e18 100644 --- a/satellite_weather_downloader/celery_app/tasks.py +++ b/satellite_weather_downloader/celery_app/tasks.py @@ -1,18 +1,18 @@ -import logging -import os -from datetime import datetime, timedelta -from turtle import update import pandas as pd import subprocess +import logging +import os +from sqlalchemy import create_engine +from celery.schedules import crontab +from datetime import datetime, timedelta from dotenv import find_dotenv, load_dotenv from satellite_weather_downloader.celery_app.celeryapp import app +from satellite_weather_downloader.utils.extract_latlons import municipios from satellite_weather_downloader.extract_reanalysis import ( download_netcdf, netcdf_to_dataframe, ) -from satellite_weather_downloader.utils.extract_latlons import municipios -from sqlalchemy import create_engine load_dotenv(find_dotenv()) @@ -66,7 +66,7 @@ def reanalysis_delete_netcdf(file: str): logging.info(f'{file.split("/")[-1]} removed.') -@app.task +@app.task(name='fetch_copernicus_weather') def reanalysis_fetch_data_daily(): today = datetime.now() @@ -103,4 +103,11 @@ def reanalysis_fetch_data_daily(): reanalysis_delete_netcdf(data) -reanalysis_fetch_data_daily() +app.conf.beat_schedule = { + # Executes every 0:01 a.m. + 'fetch-copernicus-weather-daily': { + 'task': 'fetch_copernicus_weather', + 'schedule': crontab(minute="1", hour="0"), + }, +} + diff --git a/satellite_weather_downloader/reanalysis_usage.ipynb b/satellite_weather_downloader/reanalysis_usage.ipynb index c29ba82..9b341b6 100644 --- a/satellite_weather_downloader/reanalysis_usage.ipynb +++ b/satellite_weather_downloader/reanalysis_usage.ipynb @@ -915,1013 +915,6 @@ "\n", "netcdf_to_dataframe(dados_setembro, 3304557)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b11c68eb-1d78-4a71-8e53-5572955d33dc", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "netcdf_to_dataframe(BR_09_29, 3304557)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "ec96aea2-0475-40f4-83ea-03d8796d359d", - "metadata": { - "execution": { - "iopub.execute_input": "2022-10-06T11:35:39.544275Z", - "iopub.status.busy": "2022-10-06T11:35:39.543578Z", - "iopub.status.idle": "2022-10-06T11:35:39.918047Z", - "shell.execute_reply": "2022-10-06T11:35:39.917498Z", - "shell.execute_reply.started": "2022-10-06T11:35:39.544249Z" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " | geocodigo | \n", - "temp_min | \n", - "temp_med | \n", - "temp_max | \n", - "precip_min | \n", - "precip_med | \n", - "precip_max | \n", - "pressao_min | \n", - "pressao_med | \n", - "pressao_max | \n", - "umid_min | \n", - "umid_med | \n", - "umid_max | \n", - "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
date | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
2022-09-01 | \n", - "3304557 | \n", - "13.273613 | \n", - "18.559853 | \n", - "24.848808 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1018.837463 | \n", - "1022.493530 | \n", - "1024.640625 | \n", - "45.955223 | \n", - "69.198792 | \n", - "87.169708 | \n", - "
2022-09-02 | \n", - "3304557 | \n", - "14.819794 | \n", - "20.856850 | \n", - "29.028870 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1014.810486 | \n", - "1018.093750 | \n", - "1019.984436 | \n", - "34.359180 | \n", - "65.325073 | \n", - "86.448036 | \n", - "
2022-09-03 | \n", - "3304557 | \n", - "15.996597 | \n", - "20.898968 | \n", - "27.536011 | \n", - "0.000000 | \n", - "0.001997 | \n", - "0.015141 | \n", - "1016.173645 | \n", - "1017.165344 | \n", - "1018.591309 | \n", - "46.802429 | \n", - "72.510155 | \n", - "84.202148 | \n", - "
2022-09-04 | \n", - "3304557 | \n", - "17.192703 | \n", - "18.785564 | \n", - "20.721207 | \n", - "0.047679 | \n", - "0.280212 | \n", - "0.697366 | \n", - "1020.501953 | \n", - "1023.441650 | \n", - "1025.911377 | \n", - "73.655609 | \n", - "82.952972 | \n", - "88.324532 | \n", - "
2022-09-05 | \n", - "3304557 | \n", - "17.010643 | \n", - "18.836924 | \n", - "21.863403 | \n", - "0.000469 | \n", - "0.010311 | \n", - "0.022181 | \n", - "1024.211914 | \n", - "1025.848999 | \n", - "1027.486206 | \n", - "61.256897 | \n", - "76.039719 | \n", - "85.157341 | \n", - "
2022-09-06 | \n", - "3304557 | \n", - "18.363312 | \n", - "20.415825 | \n", - "23.819839 | \n", - "0.000000 | \n", - "0.002893 | \n", - "0.012169 | \n", - "1024.383789 | \n", - "1026.630737 | \n", - "1028.510010 | \n", - "49.177200 | \n", - "67.958107 | \n", - "78.894508 | \n", - "
2022-09-07 | \n", - "3304557 | \n", - "16.214142 | \n", - "20.121052 | \n", - "25.367950 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1018.582581 | \n", - "1022.705933 | \n", - "1025.358765 | \n", - "48.671455 | \n", - "68.939568 | \n", - "84.687210 | \n", - "
2022-09-08 | \n", - "3304557 | \n", - "17.384560 | \n", - "22.722889 | \n", - "30.497002 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1012.381897 | \n", - "1015.911743 | \n", - "1019.577454 | \n", - "42.104095 | \n", - "66.718674 | \n", - "84.351418 | \n", - "
2022-09-09 | \n", - "3304557 | \n", - "19.326942 | \n", - "25.367989 | \n", - "34.232864 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1012.295715 | \n", - "1014.136719 | \n", - "1016.723206 | \n", - "29.033356 | \n", - "62.281677 | \n", - "83.088249 | \n", - "
2022-09-10 | \n", - "3304557 | \n", - "19.588982 | \n", - "25.186079 | \n", - "34.978355 | \n", - "0.000000 | \n", - "0.003159 | \n", - "0.023846 | \n", - "1012.378784 | \n", - "1013.480347 | \n", - "1016.225708 | \n", - "29.199791 | \n", - "64.219101 | \n", - "83.273239 | \n", - "
2022-09-11 | \n", - "3304557 | \n", - "18.425934 | \n", - "19.992561 | \n", - "22.084137 | \n", - "0.029567 | \n", - "0.281691 | \n", - "0.455619 | \n", - "1018.178772 | \n", - "1020.287231 | \n", - "1022.646240 | \n", - "74.584633 | \n", - "81.251511 | \n", - "87.627266 | \n", - "
2022-09-12 | \n", - "3304557 | \n", - "17.919601 | \n", - "19.163445 | \n", - "20.860703 | \n", - "0.000949 | \n", - "0.035087 | \n", - "0.088207 | \n", - "1016.396851 | \n", - "1019.633362 | \n", - "1022.351257 | \n", - "73.298241 | \n", - "81.192245 | \n", - "87.015793 | \n", - "
2022-09-13 | \n", - "3304557 | \n", - "18.434563 | \n", - "21.644608 | \n", - "26.814552 | \n", - "0.000000 | \n", - "0.014961 | \n", - "0.088687 | \n", - "1009.779968 | \n", - "1014.314575 | \n", - "1018.706299 | \n", - "60.856548 | \n", - "80.831604 | \n", - "91.894318 | \n", - "
2022-09-14 | \n", - "3304557 | \n", - "20.398476 | \n", - "21.808422 | \n", - "23.071915 | \n", - "0.000000 | \n", - "0.031085 | \n", - "0.132088 | \n", - "1012.340637 | \n", - "1014.818848 | \n", - "1017.600098 | \n", - "83.035416 | \n", - "87.801659 | \n", - "94.792259 | \n", - "
2022-09-15 | \n", - "3304557 | \n", - "19.447151 | \n", - "20.297295 | \n", - "21.050812 | \n", - "0.059367 | \n", - "0.540716 | \n", - "1.218307 | \n", - "1017.769348 | \n", - "1019.022339 | \n", - "1019.966919 | \n", - "86.509529 | \n", - "89.083611 | \n", - "91.813614 | \n", - "
2022-09-16 | \n", - "3304557 | \n", - "17.630989 | \n", - "18.590572 | \n", - "19.551071 | \n", - "0.097035 | \n", - "1.061320 | \n", - "2.530098 | \n", - "1018.670654 | \n", - "1020.726868 | \n", - "1021.991882 | \n", - "86.980721 | \n", - "90.194946 | \n", - "92.471062 | \n", - "
2022-09-17 | \n", - "3304557 | \n", - "16.272888 | \n", - "18.293110 | \n", - "20.805161 | \n", - "0.000234 | \n", - "0.102087 | \n", - "0.261315 | \n", - "1021.286194 | \n", - "1022.498840 | \n", - "1024.038086 | \n", - "62.717041 | \n", - "79.386345 | \n", - "90.183540 | \n", - "
2022-09-18 | \n", - "3304557 | \n", - "15.197975 | \n", - "18.348965 | \n", - "22.205132 | \n", - "0.000000 | \n", - "0.040458 | \n", - "0.099537 | \n", - "1018.990723 | \n", - "1021.477112 | \n", - "1023.152466 | \n", - "55.940643 | \n", - "74.839584 | \n", - "87.903412 | \n", - "
2022-09-19 | \n", - "3304557 | \n", - "14.674187 | \n", - "19.383968 | \n", - "24.434761 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1014.124512 | \n", - "1018.160400 | \n", - "1020.384338 | \n", - "51.149971 | \n", - "72.642334 | \n", - "91.275421 | \n", - "
2022-09-20 | \n", - "3304557 | \n", - "17.291397 | \n", - "21.355183 | \n", - "26.667297 | \n", - "0.000000 | \n", - "0.000194 | \n", - "0.001554 | \n", - "1008.933167 | \n", - "1012.505127 | \n", - "1014.877991 | \n", - "56.903656 | \n", - "74.727615 | \n", - "87.556297 | \n", - "
2022-09-21 | \n", - "3304557 | \n", - "20.314034 | \n", - "22.195860 | \n", - "24.929375 | \n", - "0.000000 | \n", - "0.017228 | \n", - "0.065804 | \n", - "1010.973816 | \n", - "1013.083984 | \n", - "1014.860046 | \n", - "67.046585 | \n", - "79.084778 | \n", - "87.006851 | \n", - "
2022-09-22 | \n", - "3304557 | \n", - "20.025223 | \n", - "21.726042 | \n", - "24.991608 | \n", - "0.000123 | \n", - "0.222833 | \n", - "0.462301 | \n", - "1008.205566 | \n", - "1010.355652 | \n", - "1012.726807 | \n", - "75.615273 | \n", - "87.137749 | \n", - "95.458206 | \n", - "
2022-09-23 | \n", - "3304557 | \n", - "18.911919 | \n", - "20.902691 | \n", - "22.764015 | \n", - "0.000358 | \n", - "0.082345 | \n", - "0.472436 | \n", - "1008.978149 | \n", - "1012.567688 | \n", - "1016.861816 | \n", - "62.922340 | \n", - "78.572899 | \n", - "93.371651 | \n", - "
2022-09-24 | \n", - "3304557 | \n", - "16.431877 | \n", - "19.637394 | \n", - "23.440208 | \n", - "0.000000 | \n", - "0.010432 | \n", - "0.042562 | \n", - "1019.266357 | \n", - "1020.313477 | \n", - "1022.101929 | \n", - "48.452358 | \n", - "65.652260 | \n", - "80.154778 | \n", - "
2022-09-25 | \n", - "3304557 | \n", - "17.416946 | \n", - "20.123814 | \n", - "24.380280 | \n", - "0.000000 | \n", - "0.000000 | \n", - "0.000000 | \n", - "1018.181335 | \n", - "1020.619690 | \n", - "1022.504944 | \n", - "55.855579 | \n", - "73.300682 | \n", - "83.503021 | \n", - "
2022-09-26 | \n", - "3304557 | \n", - "18.822540 | \n", - "21.300831 | \n", - "24.954575 | \n", - "0.000000 | \n", - "0.059501 | \n", - "0.475531 | \n", - "1017.050598 | \n", - "1018.390015 | \n", - "1019.746216 | \n", - "67.172211 | \n", - "79.939430 | \n", - "89.143913 | \n", - "
2022-09-27 | \n", - "3304557 | \n", - "20.163383 | \n", - "22.733868 | \n", - "26.387222 | \n", - "0.005598 | \n", - "0.222982 | \n", - "0.926020 | \n", - "1014.392578 | \n", - "1016.536499 | \n", - "1020.071899 | \n", - "69.044144 | \n", - "83.746109 | \n", - "94.025185 | \n", - "
2022-09-28 | \n", - "3304557 | \n", - "20.327026 | \n", - "21.043999 | \n", - "22.357811 | \n", - "0.000358 | \n", - "0.632614 | \n", - "1.501668 | \n", - "1015.704346 | \n", - "1017.363159 | \n", - "1019.204346 | \n", - "85.932777 | \n", - "90.595695 | \n", - "94.211685 | \n", - "
2022-09-29 | \n", - "3304557 | \n", - "18.854141 | \n", - "19.878967 | \n", - "20.482231 | \n", - "0.544429 | \n", - "1.014771 | \n", - "2.258291 | \n", - "1018.334961 | \n", - "1020.259521 | \n", - "1022.055664 | \n", - "86.878639 | \n", - "91.118866 | \n", - "94.032166 | \n", - "
\n", - " | geocodigo | \n", - "temp_min | \n", - "temp_med | \n", - "temp_max | \n", - "precip_min | \n", - "precip_med | \n", - "precip_max | \n", - "pressao_min | \n", - "pressao_med | \n", - "pressao_max | \n", - "umid_min | \n", - "umid_med | \n", - "umid_max | \n", - "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
date | \n", - "\n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " | \n", - " |
2022-01-01 | \n", - "3304557 | \n", - "23.395073 | \n", - "25.147663 | \n", - "27.565979 | \n", - "4.797475e-03 | \n", - "6.067415e-01 | \n", - "1.396402e+00 | \n", - "1008.338867 | \n", - "1009.353455 | \n", - "1010.444336 | \n", - "76.049911 | \n", - "86.932861 | \n", - "94.246536 | \n", - "
2022-01-02 | \n", - "3304557 | \n", - "24.392265 | \n", - "26.466919 | \n", - "29.889214 | \n", - "-4.656613e-07 | \n", - "2.114374e-02 | \n", - "5.662569e-02 | \n", - "1009.378784 | \n", - "1010.623657 | \n", - "1012.245605 | \n", - "64.355621 | \n", - "80.975609 | \n", - "91.027832 | \n", - "
2022-01-03 | \n", - "3304557 | \n", - "24.811615 | \n", - "27.147869 | \n", - "30.395721 | \n", - "-4.656613e-07 | \n", - "3.149961e-02 | \n", - "1.384126e-01 | \n", - "1007.511230 | \n", - "1010.032043 | \n", - "1011.380615 | \n", - "64.379768 | \n", - "80.983315 | \n", - "92.106903 | \n", - "
2022-01-04 | \n", - "3304557 | \n", - "24.652039 | \n", - "27.622200 | \n", - "31.211647 | \n", - "-4.656613e-07 | \n", - "9.774226e-02 | \n", - "4.028531e-01 | \n", - "1006.197510 | \n", - "1008.418030 | \n", - "1009.616882 | \n", - "59.122009 | \n", - "76.287140 | \n", - "89.330826 | \n", - "
2022-01-05 | \n", - "3304557 | \n", - "25.651100 | \n", - "28.077381 | \n", - "31.542862 | \n", - "-4.656613e-07 | \n", - "3.528243e-03 | \n", - "2.622057e-02 | \n", - "1004.627563 | \n", - "1005.957214 | \n", - "1007.435120 | \n", - "54.111713 | \n", - "71.372047 | \n", - "82.177231 | \n", - "
... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "
2022-06-26 | \n", - "3304557 | \n", - "18.774605 | \n", - "19.680939 | \n", - "20.940613 | \n", - "3.347057e-03 | \n", - "4.024626e-01 | \n", - "1.506641e+00 | \n", - "1020.387024 | \n", - "1021.790405 | \n", - "1023.215637 | \n", - "76.751686 | \n", - "83.051346 | \n", - "87.159851 | \n", - "
2022-06-27 | \n", - "3304557 | \n", - "16.636833 | \n", - "18.532686 | \n", - "21.597466 | \n", - "-4.656613e-07 | \n", - "3.698093e-02 | \n", - "8.323719e-02 | \n", - "1021.565002 | \n", - "1023.281372 | \n", - "1025.176147 | \n", - "60.549278 | \n", - "77.937225 | \n", - "87.060524 | \n", - "
2022-06-28 | \n", - "3304557 | \n", - "13.481377 | \n", - "18.149086 | \n", - "24.642319 | \n", - "-4.656613e-07 | \n", - "-4.656613e-07 | \n", - "-4.656613e-07 | \n", - "1017.307495 | \n", - "1020.017639 | \n", - "1022.069336 | \n", - "51.261471 | \n", - "76.206558 | \n", - "89.652519 | \n", - "
2022-06-29 | \n", - "3304557 | \n", - "15.770576 | \n", - "20.590687 | \n", - "28.492508 | \n", - "-4.656613e-07 | \n", - "-4.656613e-07 | \n", - "-4.656613e-07 | \n", - "1012.435608 | \n", - "1015.604492 | \n", - "1018.646240 | \n", - "47.530975 | \n", - "75.070236 | \n", - "89.106277 | \n", - "
2022-06-30 | \n", - "3304557 | \n", - "19.421036 | \n", - "20.724705 | \n", - "22.641853 | \n", - "-4.656613e-07 | \n", - "1.922602e-02 | \n", - "6.214879e-02 | \n", - "1014.252502 | \n", - "1018.283203 | \n", - "1020.651245 | \n", - "67.818283 | \n", - "80.456642 | \n", - "89.411125 | \n", - "
181 rows × 13 columns
\n", - "