diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1040c1c72..049bae77f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,7 @@ Changelog Development =========== +- Add TTL-based persistent caching using dogpile.cache - Add ``example/radolan.py`` and adjust documentation 0.7.0 (16.09.2020) diff --git a/poetry.lock b/poetry.lock index efe5480cd..66749e1ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,7 +15,7 @@ python-versions = "*" version = "0.7.12" [[package]] -category = "dev" +category = "main" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." name = "appdirs" optional = false @@ -168,7 +168,7 @@ description = "An easy safelist-based HTML-sanitizing tool." name = "bleach" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "3.2.0" +version = "3.2.1" [package.dependencies] packaging = "*" @@ -335,6 +335,18 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.16" +[[package]] +category = "main" +description = "A caching front-end based on the Dogpile lock." +name = "dogpile.cache" +optional = false +python-versions = ">=3.6" +version = "1.0.2" + +[package.dependencies] +decorator = ">=4.0.0" +stevedore = ">=3.0.0" + [[package]] category = "dev" description = "Discover and load entry points from installed packages." @@ -494,7 +506,7 @@ marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.1" +version = "1.7.0" [package.dependencies] zipp = ">=0.5" @@ -985,7 +997,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.8.0" [[package]] -category = "dev" +category = "main" description = "Python Build Reasonableness" name = "pbr" optional = false @@ -1105,7 +1117,7 @@ description = "Pygments is a syntax highlighting package written in Python." name = "pygments" optional = false python-versions = ">=3.5" -version = "2.7.0" +version = "2.7.1" [[package]] category = "main" @@ -1475,16 +1487,20 @@ lint = ["flake8", "mypy", "docutils-stubs"] test = ["pytest"] [[package]] -category = "dev" +category = "main" description = "Manage dynamic plugins for Python applications" name = "stevedore" optional = false python-versions = ">=3.6" -version = "2.0.1" +version = "3.2.2" [package.dependencies] pbr = ">=2.0.0,<2.1.0 || >2.1.0" +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=1.7.0" + [[package]] category = "main" description = "Hierarchical datasets for Python" @@ -1510,8 +1526,8 @@ category = "dev" description = "Terminals served to xterm.js using Tornado websockets" name = "terminado" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.8.3" +python-versions = ">=3.6" +version = "0.9.0" [package.dependencies] ptyprocess = "*" @@ -1677,7 +1693,7 @@ excel = ["openpyxl"] ipython = ["ipython", "ipython-genutils", "matplotlib"] [metadata] -content-hash = "7b5f4280a374f739b611c1c5ce0794847e05df259cf1c8401f763c3b6bf8b1b4" +content-hash = "fc60177e20eacf540af0556e0034b1523ffff8682605c1f60e252a03db2fb7eb" lock-version = "1.0" python-versions = "^3.6.1" @@ -1750,8 +1766,8 @@ black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] bleach = [ - {file = "bleach-3.2.0-py2.py3-none-any.whl", hash = "sha256:769483204d247465c0b001ead257fb86bba6944bce6fe1b6759c812cceb54e3d"}, - {file = "bleach-3.2.0.tar.gz", hash = "sha256:f9e0205cc57b558c21bdfc11034f9d96b14c4052c25be60885d94f4277c792e0"}, + {file = "bleach-3.2.1-py2.py3-none-any.whl", hash = "sha256:9f8ccbeb6183c6e6cddea37592dfb0167485c1e3b13b3363bc325aa8bda3adbd"}, + {file = "bleach-3.2.1.tar.gz", hash = "sha256:52b5919b81842b1854196eaae5ca29679a2f2e378905c346d3ca8227c2c66080"}, ] cachetools = [ {file = "cachetools-3.1.1-py2.py3-none-any.whl", hash = "sha256:428266a1c0d36dc5aca63a2d7c5942e88c2c898d72139fca0e97fdd2380517ae"}, @@ -1777,19 +1793,16 @@ cffi = [ {file = "cffi-1.14.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c687778dda01832555e0af205375d649fa47afeaeeb50a201711f9a9573323b8"}, {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, {file = "cffi-1.14.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:03d3d238cc6c636a01cf55b9b2e1b6531a7f2f4103fabb5a744231582e68ecc7"}, {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, {file = "cffi-1.14.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c2a33558fdbee3df370399fe1712d72464ce39c66436270f3664c03f94971aff"}, {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, {file = "cffi-1.14.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d9a7dc7cf8b1101af2602fe238911bcc1ac36d239e0a577831f5dac993856e9"}, @@ -1887,6 +1900,9 @@ docutils = [ {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, ] +"dogpile.cache" = [ + {file = "dogpile.cache-1.0.2.tar.gz", hash = "sha256:64fda39d25b46486a4876417ca03a4af06f35bfadba9f59613f9b3d748aa21ef"}, +] entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, @@ -1967,8 +1983,8 @@ imagesize = [ {file = "imagesize-1.2.0.tar.gz", hash = "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"}, ] importlib-metadata = [ - {file = "importlib_metadata-1.6.1-py2.py3-none-any.whl", hash = "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958"}, - {file = "importlib_metadata-1.6.1.tar.gz", hash = "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545"}, + {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, + {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, ] importlib-resources = [ {file = "importlib_resources-3.0.0-py2.py3-none-any.whl", hash = "sha256:d028f66b66c0d5732dae86ba4276999855e162a749c92620a38c1d779ed138a7"}, @@ -2018,16 +2034,19 @@ kiwisolver = [ {file = "kiwisolver-1.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:443c2320520eda0a5b930b2725b26f6175ca4453c61f739fef7a5847bd262f74"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:efcf3397ae1e3c3a4a0a0636542bcad5adad3b1dd3e8e629d0b6e201347176c8"}, {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fccefc0d36a38c57b7bd233a9b485e2f1eb71903ca7ad7adacad6c28a56d62d2"}, + {file = "kiwisolver-1.2.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:be046da49fbc3aa9491cc7296db7e8d27bcf0c3d5d1a40259c10471b014e4e0c"}, {file = "kiwisolver-1.2.0-cp36-none-win32.whl", hash = "sha256:60a78858580761fe611d22127868f3dc9f98871e6fdf0a15cc4203ed9ba6179b"}, {file = "kiwisolver-1.2.0-cp36-none-win_amd64.whl", hash = "sha256:556da0a5f60f6486ec4969abbc1dd83cf9b5c2deadc8288508e55c0f5f87d29c"}, {file = "kiwisolver-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7cc095a4661bdd8a5742aaf7c10ea9fac142d76ff1770a0f84394038126d8fc7"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c955791d80e464da3b471ab41eb65cf5a40c15ce9b001fdc5bbc241170de58ec"}, {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:603162139684ee56bcd57acc74035fceed7dd8d732f38c0959c8bd157f913fec"}, + {file = "kiwisolver-1.2.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:63f55f490b958b6299e4e5bdac66ac988c3d11b7fafa522800359075d4fa56d1"}, {file = "kiwisolver-1.2.0-cp37-none-win32.whl", hash = "sha256:03662cbd3e6729f341a97dd2690b271e51a67a68322affab12a5b011344b973c"}, {file = "kiwisolver-1.2.0-cp37-none-win_amd64.whl", hash = "sha256:4eadb361baf3069f278b055e3bb53fa189cea2fd02cb2c353b7a99ebb4477ef1"}, {file = "kiwisolver-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c31bc3c8e903d60a1ea31a754c72559398d91b5929fcb329b1c3a3d3f6e72113"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:d52b989dc23cdaa92582ceb4af8d5bcc94d74b2c3e64cd6785558ec6a879793e"}, {file = "kiwisolver-1.2.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e586b28354d7b6584d8973656a7954b1c69c93f708c0c07b77884f91640b7657"}, + {file = "kiwisolver-1.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:38d05c9ecb24eee1246391820ed7137ac42a50209c203c908154782fced90e44"}, {file = "kiwisolver-1.2.0-cp38-none-win32.whl", hash = "sha256:d069ef4b20b1e6b19f790d00097a5d5d2c50871b66d10075dab78938dc2ee2cf"}, {file = "kiwisolver-1.2.0-cp38-none-win_amd64.whl", hash = "sha256:18d749f3e56c0480dccd1714230da0f328e6e4accf188dd4e6884bdd06bf02dd"}, {file = "kiwisolver-1.2.0.tar.gz", hash = "sha256:247800260cd38160c362d211dcaf4ed0f7816afb5efe56544748b21d6ad6d17f"}, @@ -2290,6 +2309,8 @@ pillow = [ {file = "Pillow-7.2.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:5e51ee2b8114def244384eda1c82b10e307ad9778dac5c83fb0943775a653cd8"}, {file = "Pillow-7.2.0-cp38-cp38-win32.whl", hash = "sha256:725aa6cfc66ce2857d585f06e9519a1cc0ef6d13f186ff3447ab6dff0a09bc7f"}, {file = "Pillow-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:a060cf8aa332052df2158e5a119303965be92c3da6f2d93b6878f0ebca80b2f6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-macosx_10_10_x86_64.whl", hash = "sha256:9c87ef410a58dd54b92424ffd7e28fd2ec65d2f7fc02b76f5e9b2067e355ebf6"}, + {file = "Pillow-7.2.0-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:e901964262a56d9ea3c2693df68bc9860b8bdda2b04768821e4c44ae797de117"}, {file = "Pillow-7.2.0-pp36-pypy36_pp73-win32.whl", hash = "sha256:25930fadde8019f374400f7986e8404c8b781ce519da27792cbe46eabec00c4d"}, {file = "Pillow-7.2.0.tar.gz", hash = "sha256:97f9e7953a77d5a70f49b9a48da7776dc51e9b738151b22dacf101641594a626"}, ] @@ -2326,8 +2347,8 @@ pyflakes = [ {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] pygments = [ - {file = "Pygments-2.7.0-py3-none-any.whl", hash = "sha256:2df50d16b45b977217e02cba6c8422aaddb859f3d0570a88e09b00eafae89c6e"}, - {file = "Pygments-2.7.0.tar.gz", hash = "sha256:2594e8fdb06fef91552f86f4fd3a244d148ab24b66042036e64f29a291515048"}, + {file = "Pygments-2.7.1-py3-none-any.whl", hash = "sha256:307543fe65c0947b126e83dd5a61bd8acbd84abec11f43caebaf5534cbc17998"}, + {file = "Pygments-2.7.1.tar.gz", hash = "sha256:926c3f319eda178d1bd90851e4317e6d8cdb5e292a3386aac9bd75eca29cf9c7"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, @@ -2535,8 +2556,8 @@ sphinxcontrib-serializinghtml = [ {file = "sphinxcontrib_serializinghtml-1.1.4-py2.py3-none-any.whl", hash = "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a"}, ] stevedore = [ - {file = "stevedore-2.0.1-py3-none-any.whl", hash = "sha256:c4724f8d7b8f6be42130663855d01a9c2414d6046055b5a65ab58a0e38637688"}, - {file = "stevedore-2.0.1.tar.gz", hash = "sha256:609912b87df5ad338ff8e44d13eaad4f4170a65b79ae9cb0aa5632598994a1b7"}, + {file = "stevedore-3.2.2-py3-none-any.whl", hash = "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62"}, + {file = "stevedore-3.2.2.tar.gz", hash = "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0"}, ] tables = [ {file = "tables-3.6.1-2-cp36-cp36m-win32.whl", hash = "sha256:db163df08ded7804d596dee14d88397f6c55cdf4671b3992cb885c0b3890a54d"}, @@ -2566,8 +2587,8 @@ termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] terminado = [ - {file = "terminado-0.8.3-py2.py3-none-any.whl", hash = "sha256:a43dcb3e353bc680dd0783b1d9c3fc28d529f190bc54ba9a229f72fe6e7a54d7"}, - {file = "terminado-0.8.3.tar.gz", hash = "sha256:4804a774f802306a7d9af7322193c5390f1da0abb429e082a10ef1d46e6fb2c2"}, + {file = "terminado-0.9.0-py3-none-any.whl", hash = "sha256:e0eaea25d978d9a8d557f63ef23b0ec81a2f9e22b9dabfe5db4a79d8d38dd7c2"}, + {file = "terminado-0.9.0.tar.gz", hash = "sha256:7ceea4c1644b0e95588d1d2ba09c5bf365b318c26d0a13ef6a5f2751a2f6efa3"}, ] testpath = [ {file = "testpath-0.4.4-py2.py3-none-any.whl", hash = "sha256:bfcf9411ef4bf3db7579063e0546938b1edda3d69f4e1fb8756991f5951f85d4"}, diff --git a/pyproject.toml b/pyproject.toml index 4098e8654..a2c01c339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,8 +78,10 @@ dateparser = "^0.7.4" beautifulsoup4 = "^4.9.1" requests = "^2.24.0" python-dateutil = "^2.8.0" +"dogpile.cache" = "^1.0.2" +appdirs = "^1.4.4" -importlib_metadata = {version = "1.6.1", python = "<3.8"} +importlib_metadata = {version = "1.7.0", python = "<3.8"} # Optional dependencies aka. "extras" ipython = { version = "^7.10.1", optional = true } diff --git a/wetterdienst/additionals/cache.py b/wetterdienst/additionals/cache.py new file mode 100644 index 000000000..2d9be5538 --- /dev/null +++ b/wetterdienst/additionals/cache.py @@ -0,0 +1,47 @@ +import os +import logging +import platform + +import appdirs +from dogpile.cache import make_region + + +log = logging.getLogger() + +# Python on Windows has no "fcntl", which is required by the dbm backend. +# TODO: Make backend configurable, e.g. better use Redis. +platform = platform.system() +backend = "dogpile.cache.dbm" +if platform == "Windows": + backend = "dogpile.cache.memory" + +# Compute cache directory. +cache_dir = appdirs.user_cache_dir(appname="wetterdienst") +if not os.path.exists(cache_dir): + os.makedirs(cache_dir) +log.info("Cache directory is %s", cache_dir) + +# Define cache regions. +metaindex_cache = make_region().configure( + backend, + expiration_time=1 * 60, + arguments={"filename": os.path.join(cache_dir, "metaindex_1min.dbm")}, +) + +fileindex_cache_five_minutes = make_region().configure( + backend, + expiration_time=5 * 60, + arguments={"filename": os.path.join(cache_dir, "fileindex_5min.dbm")}, +) + +fileindex_cache_one_hour = make_region().configure( + backend, + expiration_time=60 * 60, + arguments={"filename": os.path.join(cache_dir, "fileindex_60min.dbm")}, +) + +payload_cache_one_hour = make_region().configure( + backend, + expiration_time=60 * 60, + arguments={"filename": os.path.join(cache_dir, "payload_60min.dbm")}, +) diff --git a/wetterdienst/data_collection.py b/wetterdienst/data_collection.py index 34d5dd376..5a2529aa0 100644 --- a/wetterdienst/data_collection.py +++ b/wetterdienst/data_collection.py @@ -125,7 +125,7 @@ def collect_climate_observations_data( continue - log.info(f"Data for {request_string} will be collected from internet.") + log.info(f"Acquiring observations data for {request_string}") remote_files = create_file_list_for_climate_observations( [station_id], parameter, time_resolution, period_type @@ -273,9 +273,7 @@ def collect_radolan_data( continue except FileNotFoundError: - log.info( - f"RADOLAN data for {str(date_time)} will be collected from internet" - ) + log.info(f"Acquiring RADOLAN data for {str(date_time)}") remote_radolan_file_path = create_filepath_for_radolan( date_time, time_resolution diff --git a/wetterdienst/download/download.py b/wetterdienst/download/download.py index 50d97d214..e8db956dd 100644 --- a/wetterdienst/download/download.py +++ b/wetterdienst/download/download.py @@ -9,10 +9,10 @@ from io import BytesIO from concurrent.futures import ThreadPoolExecutor from datetime import datetime -from functools import lru_cache from requests.exceptions import InvalidURL +from wetterdienst.additionals.cache import payload_cache_one_hour from wetterdienst.constants.access_credentials import DWDCDCBase from wetterdienst.download.download_services import download_file_from_dwd from wetterdienst.enumerations.datetime_format_enumeration import DatetimeFormat @@ -32,16 +32,12 @@ def download_climate_observations_data_parallel( """ with ThreadPoolExecutor() as executor: - files_in_bytes = executor.map( - _download_climate_observations_data_parallel, remote_files - ) + files_in_bytes = executor.map(_download_climate_observations_data, remote_files) return list(zip(remote_files, files_in_bytes)) -def _download_climate_observations_data_parallel( - remote_file: Union[str, Path] -) -> BytesIO: +def _download_climate_observations_data(remote_file: Union[str, Path]) -> BytesIO: """ This function downloads the station data for which the link is provided by the 'select_dwd' function. It checks the shortened filepath (just @@ -56,6 +52,11 @@ def _download_climate_observations_data_parallel( stores data on local file system """ + return BytesIO(__download_climate_observations_data(remote_file=remote_file)) + + +@payload_cache_one_hour.cache_on_arguments() +def __download_climate_observations_data(remote_file: Union[str, Path]) -> bytes: try: zip_file = download_file_from_dwd(remote_file, DWDCDCBase.CLIMATE_OBSERVATIONS) except InvalidURL as e: @@ -74,7 +75,7 @@ def _download_climate_observations_data_parallel( for file in archive_files: # If found file load file in bytes, close zipfile and return bytes if file.startswith(PRODUCT_FILE_IDENTIFIER): - file_in_bytes = BytesIO(zip_file_opened.open(file).read()) + file_in_bytes = zip_file_opened.open(file).read() zip_file_opened.close() @@ -113,7 +114,7 @@ def download_radolan_data( return _extract_radolan_data(date_time, archive_in_bytes) -@lru_cache(maxsize=750) +@payload_cache_one_hour.cache_on_arguments() def _download_radolan_data(remote_radolan_filepath: str) -> BytesIO: """ Function (cached) that downloads the RADOLAN file diff --git a/wetterdienst/download/download_services.py b/wetterdienst/download/download_services.py index 356adef7c..8ba0e05be 100644 --- a/wetterdienst/download/download_services.py +++ b/wetterdienst/download/download_services.py @@ -1,6 +1,7 @@ """ **DWD download utilities** """ +import logging from io import BytesIO from pathlib import PurePosixPath from typing import Union @@ -9,6 +10,8 @@ from wetterdienst.download.https_handling import create_dwd_session from wetterdienst.file_path_handling.path_handling import build_dwd_cdc_data_path +logger = logging.getLogger(__name__) + def download_file_from_dwd( filepath: Union[PurePosixPath, str], cdc_base: DWDCDCBase @@ -24,7 +27,9 @@ def download_file_from_dwd( """ dwd_session = create_dwd_session() - r = dwd_session.get(build_dwd_cdc_data_path(filepath, cdc_base)) + url = build_dwd_cdc_data_path(filepath, cdc_base) + logger.info(f"Downloading resource {url}") + r = dwd_session.get(url) r.raise_for_status() return BytesIO(r.content) diff --git a/wetterdienst/indexing/file_index_creation.py b/wetterdienst/indexing/file_index_creation.py index 1d21e08e6..d87ced0bd 100644 --- a/wetterdienst/indexing/file_index_creation.py +++ b/wetterdienst/indexing/file_index_creation.py @@ -1,10 +1,13 @@ """ file index creation for available DWD station data """ import re -from functools import lru_cache import pandas as pd from dateparser import parse +from wetterdienst.additionals.cache import ( + fileindex_cache_five_minutes, + fileindex_cache_one_hour, +) from wetterdienst.constants.access_credentials import ( DWD_CDC_PATH, DWDCDCBase, @@ -26,7 +29,7 @@ ) -@lru_cache(maxsize=None) +@fileindex_cache_five_minutes.cache_on_arguments() def create_file_index_for_climate_observations( parameter: Parameter, time_resolution: TimeResolution, period_type: PeriodType ) -> pd.DataFrame: @@ -63,7 +66,7 @@ def create_file_index_for_climate_observations( ] -@lru_cache(maxsize=None) +@fileindex_cache_one_hour.cache_on_arguments() def create_file_index_for_radolan(time_resolution: TimeResolution) -> pd.DataFrame: """ Function used to create a file index for the RADOLAN product. The file index will @@ -146,5 +149,5 @@ def _create_file_index_for_dwd_server( def reset_file_index_cache() -> None: """ Function to reset the cached file index for all kinds of parameters """ - create_file_index_for_climate_observations.cache_clear() - create_file_index_for_radolan.cache_clear() + fileindex_cache_five_minutes.invalidate() + fileindex_cache_one_hour.invalidate() diff --git a/wetterdienst/indexing/meta_index_creation.py b/wetterdienst/indexing/meta_index_creation.py index 6ab14170b..3cc3cd297 100644 --- a/wetterdienst/indexing/meta_index_creation.py +++ b/wetterdienst/indexing/meta_index_creation.py @@ -4,12 +4,12 @@ from pathlib import PurePosixPath from typing import Tuple, List from concurrent.futures import ThreadPoolExecutor -from functools import lru_cache import datetime as dt import pandas as pd from requests.exceptions import InvalidURL +from wetterdienst.additionals.cache import metaindex_cache from wetterdienst.constants.access_credentials import DWDCDCBase from wetterdienst.constants.column_name_mapping import ( GERMAN_TO_ENGLISH_COLUMNS_MAPPING, @@ -60,7 +60,7 @@ ] -@lru_cache(maxsize=None) +@metaindex_cache.cache_on_arguments() def create_meta_index_for_climate_observations( parameter: Parameter, time_resolution: TimeResolution, period_type: PeriodType ) -> pd.DataFrame: @@ -340,4 +340,4 @@ def _parse_zipped_data_into_df(file: BytesIO) -> pd.DataFrame: def reset_meta_index_cache() -> None: """ Function to reset cache of meta index """ - create_meta_index_for_climate_observations.cache_clear() + metaindex_cache.invalidate()