From baa1cefc8d261404c6b1b8bece0de570ebe9f716 Mon Sep 17 00:00:00 2001 From: jonavellecuerdo Date: Fri, 1 Nov 2024 16:19:09 -0400 Subject: [PATCH 1/2] Add option to download input files using a local MinIO server Why these changes are being introduced: * Downloading extract files improves the performance of the app by reducing requests sent to AWS S3 and avoiding repeated downloads of extract files used across multiple container runs. Having extract files available on local disk also minimizes the occurence of network issues or AWS credentials timing out during a transform. These changes introduces a locally hosted MinIO server to act as a "local S3 bucket" as part of the A/B diff workflow. How this addresses that need: * Add a Docker Compose YAML file to run local MinIO server * Add Makefile commands for starting and stopping local MinIO server * Add option '--download-files' to run-diff CLI command * Implement download_input_files core function * Update run_ab_transforms to suport use of local MinIO server Side effects of this change: * None Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/TIMX-353 --- Makefile | 8 + Pipfile | 1 + Pipfile.lock | 402 +++++++++++++---------- README.md | 37 +++ abdiff/cli.py | 12 +- abdiff/config.py | 21 ++ abdiff/core/__init__.py | 2 + abdiff/core/download_input_files.py | 85 +++++ abdiff/core/run_ab_transforms.py | 39 ++- abdiff/helpers/minio/docker-compose.yaml | 24 ++ 10 files changed, 445 insertions(+), 186 deletions(-) create mode 100644 abdiff/core/download_input_files.py create mode 100644 abdiff/helpers/minio/docker-compose.yaml diff --git a/Makefile b/Makefile index dc65248..1c9b1eb 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) +MINIO_COMPOSE_FILE=abdiff/helpers/minio/docker-compose.yaml help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ @@ -54,3 +55,10 @@ black-apply: # Apply changes with 'black' ruff-apply: # Resolve 'fixable errors' with 'ruff' pipenv run ruff check --fix . + +# Development commands +start-minio-server: + docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) up -d + +stop-minio-server: + docker compose -f $(MINIO_COMPOSE_FILE) stop \ No newline at end of file diff --git a/Pipfile b/Pipfile index 8f87bda..8880b2f 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,7 @@ deepdiff = "*" [dev-packages] black = "*" +boto3-stubs = {version = "*", extras = ["s3"]} coveralls = "*" freezegun = "*" ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1ff4aa9..9f0c47d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4811e3bc6c1ebe8265070fcf866a2b88abf2ed5c77182164fdb978c2758bac23" + "sha256": "6742438ee4103c6b5e0b05af917ed22014aca7fc30a75dfce5d27a84fef01483" }, "pipfile-spec": 6, "requires": { @@ -26,20 +26,20 @@ }, "boto3": { "hashes": [ - "sha256:a9c0955df0b52b43749d81bde159343a40ea2a3537a46049336fe8193871b18e", - "sha256:f4124548bb831e13504e805f2fbbfcee06df42fffea0655862c6eb9b95d6d1be" + "sha256:2d5e160b614db55fbee7981001c54476cb827c441cef65b2fcb2c52a62019909", + "sha256:7d9c359bbbc858a60b51c86328db813353c8bd1940212cdbd0a7da835291c2e1" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.35.53" + "version": "==1.35.54" }, "botocore": { "hashes": [ - "sha256:12869640f2f9fab3152ea312a6906d5bc6ae15522cc74b6367ee1c273269a28b", - "sha256:e610ae076ad1eaed5680d3990493659bbabdffd67b15c61d8373a23e4bc41062" + "sha256:131bb59ce59c8a939b31e8e647242d70cf11d32d4529fa4dca01feea1e891a76", + "sha256:9cca1811094b6cdc144c2c063a3ec2db6d7c88194b04d4277cd34fc8e3473aff" ], "markers": "python_version >= '3.8'", - "version": "==1.35.53" + "version": "==1.35.54" }, "certifi": { "hashes": [ @@ -262,62 +262,62 @@ }, "duckdb": { "hashes": [ - "sha256:0107de622fe208142a1108263a03c43956048dcc99be3702d8e5d2aeaf99554c", - "sha256:099d99dd48d6e4682a3dd6233ceab73d977ebe1a87afaac54cf77c844e24514a", - "sha256:181edb1973bd8f493bcb6ecfa035f1a592dff4667758592f300619012ba251c0", - "sha256:19142a77e72874aeaa6fda30aeb13612c6de5e8c60fbcc3392cea6ef0694eeaf", - "sha256:19386aa09f0d6f97634ba2972096d1c80d880176dfb0e949eadc91c98262a663", - "sha256:1bba58459ad897a78c4e478a097626fc266459a40338cecc68a49a8d5dc72fb7", - "sha256:25889e6e29b87047b1dd56385ac08156e4713c59326cc6fff89657d01b2c417b", - "sha256:312570fa5277c3079de18388b86c2d87cbe1044838bb152b235c0227581d5d42", - "sha256:492b1d86a696428bd3f14dc1c7c3230e2dbca8978f288be64b04a26e0e00fad5", - "sha256:4ce949f1d7999aa6a046eb64067eee41d4c5c2872ba4fa408c9947742d0c7231", - "sha256:564166811c68d9c7f9911eb707ad32ec9c2507b98336d894fbe658b85bf1c697", - "sha256:568439ea4fce8cb72ec1f767cd510686a9e7e29a011fc7c56d990059a6e94e48", - "sha256:578e0953e4d8ba8da0cd69fb2930c45f51ce47d213b77d8a4cd461f9c0960b87", - "sha256:623cb1952466aae5907af84107bcdec25a5ca021a8b6441e961f41edc724f6f2", - "sha256:6858e10c60ff7e70e61d3dd53d2545c8b2609942e45fd6de38cd0dee52932de3", - "sha256:695dcbc561374b126e86659709feadf883c9969ed718e94713edd4ba15d16619", - "sha256:6c37b039f6d6fed14d89450f5ccf54922b3304192d7412e12d6cc8d9e757f7a2", - "sha256:6ca722738fa9eb6218619740631de29acfdd132de6f6a6350fee5e291c2f6117", - "sha256:7111fd3e7b334a7be383313ce29918b7c643e4f6ef44d6d63c3ab3fa6716c114", - "sha256:72b5eb5762c1a5e68849c7143f3b3747a9f15c040e34e41559f233a1569ad16f", - "sha256:74974f2d7210623a5d61b1fb0cb589c6e5ffcbf7dbb757a04c5ba24adcfc8cac", - "sha256:7ca967c5a57b1d0cb0fd5e539ab24110e5a59dcbedd365bb2dc80533d6e44a8d", - "sha256:83372b1b411086cac01ab2071122772fa66170b1b41ddbc37527464066083668", - "sha256:84103373e818758dfa361d27781d0f096553843c5ffb9193260a0786c5248270", - "sha256:87e972bd452eeeab197fe39dcaeecdb7c264b1f75a0ee67e532e235fe45b84df", - "sha256:89164a2d29d56605a95ee5032aa415dd487028c4fd3e06d971497840e74c56e7", - "sha256:8a01fae9604a54ecbc26e7503c522311f15afbd2870e6d8f6fbef4545dfae550", - "sha256:8a09610f780857677725897856f8cdf3cafd8a991f871e6cb8ba88b2dbc8d737", - "sha256:91e7f99cf5cab1d26f92cb014429153497d805e79689baa44f4c4585a8cb243f", - "sha256:9283dcca87c3260eb631a99d738fa72b8545ed45b475bc72ad254f7310e14284", - "sha256:9a6b73e70b73c8df85da383f6e557c03cad5c877868b9a7e41715761e8166c1e", - "sha256:9b4c6b6a08180261d98330d97355503961a25ca31cd9ef296e0681f7895b4a2c", - "sha256:9ba6d1f918e6ca47a368a0c32806016405cb9beb2c245806b0ca998f569d2bdf", - "sha256:ada29be1e889f486c6cf1f6dffd15463e748faf361f33996f2e862779edc24a9", - "sha256:be86e586ca7af7e807f72479a2b8d0983565360b19dbda4ef8a9d7b3909b8e2c", - "sha256:bfdfd23e2bf58014ad0673973bd0ed88cd048dfe8e82420814a71d7d52ef2288", - "sha256:c0f0ddac0482f0f3fece54d720d13819e82ae26c01a939ffa66a87be53f7f665", - "sha256:c796d33f1e5a0c8c570d22da0c0b1db8578687e427029e1ce2c8ce3f9fffa6a3", - "sha256:c8232861dc8ec6daa29067056d5a0e5789919f2ab22ab792787616d7cd52f02a", - "sha256:d395a3bf510bf24686821eec15802624797dcb33e8f14f8a7cc8e17d909474af", - "sha256:d8caaf43909e49537e26df51d80d075ae2b25a610d28ed8bd31d6ccebeaf3c65", - "sha256:d9fc0b550f96901fa7e76dc70a13f6477ad3e18ef1cb21d414c3a5569de3f27e", - "sha256:db37441deddfee6ac35a0c742d2f9e90e4e50b9e76d586a060d122b8fc56dada", - "sha256:e26422a3358c816d764639070945b73eef55d1b4df990989e3492c85ef725c21", - "sha256:e3e6300b7ccaf64b609f4f0780a6e1d25ab8cf34cceed46e62c35b6c4c5cb63b", - "sha256:e8c766b87f675c76d6d17103bf6fb9fb1a9e2fcb3d9b25c28bbc634bde31223e", - "sha256:e9e8387bcc9a591ad14011ddfec0d408d1d9b1889c6c9b495a04c7016a24b9b3", - "sha256:efec169b3fe0b821e3207ba3e445f227d42dd62b4440ff79c37fa168a4fc5a71", - "sha256:f5c0996988a70dd3bc8111d9b9aeab7e38ed1999a52607c5f1b528e362b4dd1c", - "sha256:f87edaf20001530e63a4f7bda13b55dc3152d7171226915f2bf34e0813c8759e", - "sha256:f8c5ff4970403ed3ff0ac71fe0ce1e6be3199df9d542afc84c424b444ba4ffe8", - "sha256:fd800f75728727fe699ed1eb22b636867cf48c9dd105ee88b977e20c89df4509" + "sha256:00cca22df96aa3473fe4584f84888e2cf1c516e8c2dd837210daec44eadba586", + "sha256:08935700e49c187fe0e9b2b86b5aad8a2ccd661069053e38bfaed3b9ff795efd", + "sha256:0897f83c09356206ce462f62157ce064961a5348e31ccb2a557a7531d814e70e", + "sha256:09c68522c30fc38fc972b8a75e9201616b96ae6da3444585f14cf0d116008c95", + "sha256:0a55169d2d2e2e88077d91d4875104b58de45eff6a17a59c7dc41562c73df4be", + "sha256:0ba6baa0af33ded836b388b09433a69b8bec00263247f6bf0a05c65c897108d3", + "sha256:183ac743f21c6a4d6adfd02b69013d5fd78e5e2cd2b4db023bc8a95457d4bc5d", + "sha256:1aa3abec8e8995a03ff1a904b0e66282d19919f562dd0a1de02f23169eeec461", + "sha256:1c0226dc43e2ee4cc3a5a4672fddb2d76fd2cf2694443f395c02dd1bea0b7fce", + "sha256:1d9ab6143e73bcf17d62566e368c23f28aa544feddfd2d8eb50ef21034286f24", + "sha256:2141c6b28162199999075d6031b5d63efeb97c1e68fb3d797279d31c65676269", + "sha256:252d9b17d354beb9057098d4e5d5698e091a4f4a0d38157daeea5fc0ec161670", + "sha256:25fb02629418c0d4d94a2bc1776edaa33f6f6ccaa00bd84eb96ecb97ae4b50e9", + "sha256:2f073d15d11a328f2e6d5964a704517e818e930800b7f3fa83adea47f23720d3", + "sha256:35c420f58abc79a68a286a20fd6265636175fadeca1ce964fc8ef159f3acc289", + "sha256:4ebf5f60ddbd65c13e77cddb85fe4af671d31b851f125a4d002a313696af43f1", + "sha256:4f0e2e5a6f5a53b79aee20856c027046fba1d73ada6178ed8467f53c3877d5e0", + "sha256:51c6d79e05b4a0933672b1cacd6338f882158f45ef9903aef350c4427d9fc898", + "sha256:51e7dbd968b393343b226ab3f3a7b5a68dee6d3fe59be9d802383bf916775cb8", + "sha256:5ace6e4b1873afdd38bd6cc8fcf90310fb2d454f29c39a61d0c0cf1a24ad6c8d", + "sha256:5d57776539211e79b11e94f2f6d63de77885f23f14982e0fac066f2885fcf3ff", + "sha256:6411e21a2128d478efbd023f2bdff12464d146f92bc3e9c49247240448ace5a6", + "sha256:647f17bd126170d96a38a9a6f25fca47ebb0261e5e44881e3782989033c94686", + "sha256:68c3a46ab08836fe041d15dcbf838f74a990d551db47cb24ab1c4576fc19351c", + "sha256:77f26884c7b807c7edd07f95cf0b00e6d47f0de4a534ac1706a58f8bc70d0d31", + "sha256:7c71169fa804c0b65e49afe423ddc2dc83e198640e3b041028da8110f7cd16f7", + "sha256:80158f4c7c7ada46245837d5b6869a336bbaa28436fbb0537663fa324a2750cd", + "sha256:872d38b65b66e3219d2400c732585c5b4d11b13d7a36cd97908d7981526e9898", + "sha256:8ee97ec337794c162c0638dda3b4a30a483d0587deda22d45e1909036ff0b739", + "sha256:911d58c22645bfca4a5a049ff53a0afd1537bc18fedb13bc440b2e5af3c46148", + "sha256:9c619e4849837c8c83666f2cd5c6c031300cd2601e9564b47aa5de458ff6e69d", + "sha256:9d0767ada9f06faa5afcf63eb7ba1befaccfbcfdac5ff86f0168c673dd1f47aa", + "sha256:9e3f5cd604e7c39527e6060f430769b72234345baaa0987f9500988b2814f5e4", + "sha256:a1f83c7217c188b7ab42e6a0963f42070d9aed114f6200e3c923c8899c090f16", + "sha256:a1fa0c502f257fa9caca60b8b1478ec0f3295f34bb2efdc10776fc731b8a6c5f", + "sha256:a30dd599b8090ea6eafdfb5a9f1b872d78bac318b6914ada2d35c7974d643640", + "sha256:a433ae9e72c5f397c44abdaa3c781d94f94f4065bcbf99ecd39433058c64cb38", + "sha256:a4748635875fc3c19a7320a6ae7410f9295557450c0ebab6d6712de12640929a", + "sha256:b74e121ab65dbec5290f33ca92301e3a4e81797966c8d9feef6efdf05fc6dafd", + "sha256:c443d3d502335e69fc1e35295fcfd1108f72cb984af54c536adfd7875e79cee5", + "sha256:c5336939d83837af52731e02b6a78a446794078590aa71fd400eb17f083dda3e", + "sha256:cddc6c1a3b91dcc5f32493231b3ba98f51e6d3a44fe02839556db2b928087378", + "sha256:d08308e0a46c748d9c30f1d67ee1143e9c5ea3fbcccc27a47e115b19e7e78aa9", + "sha256:d5724fd8a49e24d730be34846b814b98ba7c304ca904fbdc98b47fa95c0b0cee", + "sha256:e4ef7ba97a65bd39d66f2a7080e6fb60e7c3e41d4c1e19245f90f53b98e3ac32", + "sha256:e59087dbbb63705f2483544e01cccf07d5b35afa58be8931b224f3221361d537", + "sha256:e86006958e84c5c02f08f9b96f4bc26990514eab329b1b4f71049b3727ce5989", + "sha256:ecb1dc9062c1cc4d2d88a5e5cd8cc72af7818ab5a3c0f796ef0ffd60cfd3efb4", + "sha256:eeacb598120040e9591f5a4edecad7080853aa8ac27e62d280f151f8c862afa3", + "sha256:f549af9f7416573ee48db1cf8c9d27aeed245cb015f4b4f975289418c6cf7320", + "sha256:f58db1b65593ff796c8ea6e63e2e144c944dd3d51c8d8e40dffa7f41693d35d3", + "sha256:f9b47036945e1db32d70e414a10b1593aec641bd4c5e2056873d971cc21e978b" ], "index": "pypi", "markers": "python_full_version >= '3.7.0'", - "version": "==1.1.2" + "version": "==1.1.3" }, "flask": { "hashes": [ @@ -529,62 +529,64 @@ }, "numpy": { "hashes": [ - "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8", - "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466", - "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35", - "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c", - "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4", - "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6", - "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0", - "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7", - "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a", - "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a", - "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e", - "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62", - "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2", - "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5", - "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee", - "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe", - "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a", - "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e", - "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf", - "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c", - "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3", - "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86", - "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df", - "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98", - "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d", - "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2", - "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146", - "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550", - "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8", - "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb", - "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e", - "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d", - "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366", - "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0", - "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db", - "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe", - "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426", - "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952", - "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03", - "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f", - "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7", - "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b", - "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17", - "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5", - "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1", - "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142", - "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884", - "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a", - "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9", - "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445", - "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1", - "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1", - "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648" + "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe", + "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0", + "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48", + "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a", + "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564", + "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958", + "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17", + "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0", + "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee", + "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b", + "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4", + "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4", + "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6", + "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4", + "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d", + "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f", + "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f", + "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f", + "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56", + "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9", + "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd", + "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23", + "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed", + "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a", + "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098", + "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1", + "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512", + "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f", + "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09", + "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f", + "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc", + "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8", + "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0", + "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761", + "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef", + "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5", + "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e", + "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b", + "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d", + "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43", + "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c", + "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41", + "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff", + "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408", + "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2", + "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9", + "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57", + "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb", + "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9", + "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3", + "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a", + "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0", + "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e", + "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598", + "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4" ], "markers": "python_version >= '3.12'", - "version": "==2.1.2" + "version": "==2.1.3" }, "orderly-set": { "hashes": [ @@ -741,7 +743,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pytz": { @@ -772,7 +774,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "types-cffi": { @@ -811,11 +813,11 @@ }, "types-setuptools": { "hashes": [ - "sha256:2949913a518d5285ce00a3b7d88961c80a6e72ffb8f3da0a3f5650ea533bd45e", - "sha256:6721ac0f1a620321e2ccd87a9a747c4a383dc381f78d894ce37f2455b45fcf1c" + "sha256:080883032fb26dfb91c748f69e4efbb202aeeb3f9e95e68bf7352b7ab215069f", + "sha256:c5a98b3308fea29d26228978bd83f6ea262d2fd4d967e34e8e49d04708bb3022" ], "markers": "python_version >= '3.8'", - "version": "==75.2.0.20241025" + "version": "==75.3.0.20241105" }, "tzdata": { "hashes": [ @@ -835,11 +837,11 @@ }, "werkzeug": { "hashes": [ - "sha256:8cd39dfbdfc1e051965f156163e2974e52c210f130810e9ad36858f0fd3edad4", - "sha256:a71124d1ef06008baafa3d266c02f56e1836a5984afd6dd6c9230669d60d9fb5" + "sha256:4f7d1a5de312c810a8a2c6f0b47e9f6a7cffb7c8322def35e4d4d9841ff85597", + "sha256:f471a4cd167233077e9d2a8190c3471c5bc520c636a9e3c1e9300c33bced03bc" ], "markers": "python_version >= '3.9'", - "version": "==3.1.1" + "version": "==3.1.2" } }, "develop": { @@ -879,6 +881,25 @@ "markers": "python_version >= '3.9'", "version": "==24.10.0" }, + "boto3-stubs": { + "extras": [ + "s3" + ], + "hashes": [ + "sha256:17519739590614d50a4bfb82d10e02a9bf324ab87e90d54ecddf68044ba219b7", + "sha256:ecf6cce737abc9bc212d48238021ff4aa3fb20ed887ecdee72a750bd329c902e" + ], + "markers": "python_version >= '3.8'", + "version": "==1.35.54" + }, + "botocore-stubs": { + "hashes": [ + "sha256:26ba65907eed959dddc644ab1cd72e3a2cc9761dad79e0b45ff3b8676c47e5ec", + "sha256:49e28813324308bfc5a92bde118df5c9c41a01237eef1e1628891770f3f68a94" + ], + "markers": "python_version >= '3.8'", + "version": "==1.35.54" + }, "certifi": { "hashes": [ "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", @@ -1229,6 +1250,13 @@ "markers": "python_version >= '3.8'", "version": "==1.13.0" }, + "mypy-boto3-s3": { + "hashes": [ + "sha256:34d19dfba400f5b9bd6b64f09eb8f8eedef60545b410a3753fe99fec0c41ba78", + "sha256:f0087a3765d103b2db565cd8065ebc2b0f70f2dd4e92c132f64b8945dd869940" + ], + "version": "==1.35.46" + }, "mypy-extensions": { "hashes": [ "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", @@ -1247,62 +1275,64 @@ }, "numpy": { "hashes": [ - "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8", - "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466", - "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35", - "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c", - "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4", - "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6", - "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0", - "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7", - "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a", - "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a", - "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e", - "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62", - "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2", - "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5", - "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee", - "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe", - "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a", - "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e", - "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf", - "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c", - "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3", - "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86", - "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df", - "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98", - "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d", - "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2", - "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146", - "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550", - "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8", - "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb", - "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e", - "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d", - "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366", - "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0", - "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db", - "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe", - "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426", - "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952", - "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03", - "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f", - "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7", - "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b", - "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17", - "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5", - "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1", - "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142", - "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884", - "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a", - "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9", - "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445", - "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1", - "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1", - "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648" + "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe", + "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0", + "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48", + "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a", + "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564", + "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958", + "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17", + "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0", + "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee", + "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b", + "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4", + "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4", + "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6", + "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4", + "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d", + "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f", + "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f", + "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f", + "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56", + "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9", + "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd", + "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23", + "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed", + "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a", + "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098", + "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1", + "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512", + "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f", + "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09", + "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f", + "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc", + "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8", + "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0", + "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761", + "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef", + "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5", + "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e", + "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b", + "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d", + "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43", + "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c", + "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41", + "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff", + "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408", + "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2", + "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9", + "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57", + "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb", + "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9", + "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3", + "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a", + "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0", + "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e", + "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598", + "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4" ], "markers": "python_version >= '3.12'", - "version": "==2.1.2" + "version": "==2.1.3" }, "packaging": { "hashes": [ @@ -1472,7 +1502,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pyyaml": { @@ -1581,7 +1611,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "stack-data": { @@ -1599,6 +1629,14 @@ "markers": "python_version >= '3.8'", "version": "==5.14.3" }, + "types-awscrt": { + "hashes": [ + "sha256:3fd1edeac923d1956c0e907c973fb83bda465beae7f054716b371b293f9b5fdc", + "sha256:517d9d06f19cf58d778ca90ad01e52e0489466bf70dcf78c7f47f74fdf151a60" + ], + "markers": "python_version >= '3.8'", + "version": "==0.23.0" + }, "types-pytz": { "hashes": [ "sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7", @@ -1607,6 +1645,14 @@ "markers": "python_version >= '3.8'", "version": "==2024.2.0.20241003" }, + "types-s3transfer": { + "hashes": [ + "sha256:d34c5a82f531af95bb550927136ff5b737a1ed3087f90a59d545591dfde5b4cc", + "sha256:f761b2876ac4c208e6c6b75cdf5f6939009768be9950c545b11b0225e7703ee7" + ], + "markers": "python_version >= '3.8'", + "version": "==0.10.3" + }, "typing-extensions": { "hashes": [ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", diff --git a/README.md b/README.md index c97e327..2b8bcbb 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,38 @@ Compare transformed TIMDEX records from two versions (A,B) of Transmogrifier. - To lint the repo: `make lint` - To run the app: `pipenv run abdiff --help` +### Running a Local MinIO Server + +TIMDEX extract files from S3 (i.e., input files to use in transformations) can be downloaded to a local MinIO server hosted with a Docker container. [MinIO is an object storage solution that provides an Amazon Web Services S3-compatible API and supports all core S3 features](https://min.io/docs/minio/kubernetes/upstream/). Downloading extract files improves the runtime of a diff by reducing the number of requests sent to S3 and avoids repeated downloads of extract files. + +1. Configure your `.env` file. In addition to the [required environment variables](#required), the following environment variables must also be set: + + ```text + MINIO_S3_LOCAL_STORAGE= + TIMDEX_BUCKET= + ``` + + Note: There are additional variables required by the Local MinIO server (see vars prefixed with "MINIO" in [optional environment variables](#optional)). For these variables, defaults are provided in [abdiff.config](abdiff/config.py). + +2. Create an AWS profile `minio`. When prompted for an "AWS Access Key ID" and "AWS Secret Access Key", pass the values set for the `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD` environment variables, respectively. + ```shell + aws configure --profile minio + ``` + +3. Launch a local MinIO server via Docker container by running the Makefile command: + ```shell + make start-minio-server + ``` + + The API is accessible at: http://127.0.0.1:9000. + The WebUI is accessible at: http://127.0.0.1:9001. + +4. On your browser, navigate to the WebUI and sign into the local MinIO server. Create a bucket in the local MinIO server named after the S3 bucket containing the TIMDEX extract files that will be used in the A/B Diff. + +5. Proceed with A/B Diff CLI commands as needed! + +Once a diff run is complete, you can stop the local MinIO server using the Makefile command: `make stop-minio-server`. If you're planning to run another diff using the same files -- good news! All you have to do is restart the local MinIO server. Your data will persist as long as the files exist in the directory you specified for `MINIO_S3_LOCAL_STORAGE`. + ## Concepts A **Job** in `abdiff` represents the A/B test for comparing the results from two versions of Transmogrifier. When a job is first created, a working directory and a JSON file `job.json` with an initial set of configurations is created. @@ -90,6 +122,11 @@ AWS_SESSION_TOKEN=# passed to Transmogrifier containers for use ### Optional ```text +MINIO_S3_LOCAL_STORAGE=# full file system path to the directory where MinIO stores its object data on the local disk +MINIO_S3_URL=# endpoint for MinIO server API; default is "http://localhost:9000/" +MINIO_S3_CONTAINER_URL=# endpoint for the MinIO server when acccessed from inside a Docker container; default is "http://host.docker.internal:9000/" +MINIO_ROOT_USER=# username for root user account for MinIO server +MINIO_ROOT_PASSWORD=# password for root user account MinIO server WEBAPP_HOST=# host for flask webapp WEBAPP_PORT=# port for flask webapp TRANSMOGRIFIER_MAX_WORKERS=# max number of Transmogrifier containers to run in parallel; default is 6 diff --git a/abdiff/cli.py b/abdiff/cli.py index c42aaef..77d78fb 100644 --- a/abdiff/cli.py +++ b/abdiff/cli.py @@ -14,6 +14,7 @@ calc_ab_diffs, calc_ab_metrics, collate_ab_transforms, + download_input_files, init_run, run_ab_transforms, ) @@ -148,7 +149,12 @@ def init_job( help="Message to describe Run.", default="Not provided.", ) -def run_diff(job_directory: str, input_files: str, message: str) -> None: +@click.option( + "--download-files", is_flag=True, help="Pass to skip download of extract files" +) +def run_diff( + job_directory: str, input_files: str, message: str, *, download_files: bool +) -> None: job_data = read_job_json(job_directory) run_directory = init_run(job_directory, message=message) @@ -160,11 +166,15 @@ def run_diff(job_directory: str, input_files: str, message: str) -> None: else: input_files_list = [filepath.strip() for filepath in input_files.split(",")] + if download_files: + download_input_files(input_files_list) + ab_transformed_file_lists = run_ab_transforms( run_directory=run_directory, image_tag_a=job_data["image_tag_a"], image_tag_b=job_data["image_tag_b"], input_files=input_files_list, + use_local_s3=download_files, ) collated_dataset_path = collate_ab_transforms( run_directory=run_directory, diff --git a/abdiff/config.py b/abdiff/config.py index 66eaee1..abd5abe 100644 --- a/abdiff/config.py +++ b/abdiff/config.py @@ -11,6 +11,11 @@ class Config: "WORKSPACE", ) OPTIONAL_ENV_VARS = ( + "MINIO_S3_LOCAL_STORAGE", + "MINIO_S3_URL", + "MINIO_S3_CONTAINER_URL", + "MINIO_ROOT_USER", + "MINIO_ROOT_PASSWORD", "WEBAPP_HOST", "WEBAPP_PORT", "TRANSMOGRIFIER_MAX_WORKERS", @@ -25,6 +30,22 @@ def __getattr__(self, name: str) -> Any: # noqa: ANN401 message = f"'{name}' not a valid configuration variable" raise AttributeError(message) + @property + def minio_s3_url(self) -> str: + return self.MINIO_S3_URL or "http://localhost:9000/" + + @property + def minio_s3_container_url(self) -> str: + return self.MINIO_S3_CONTAINER_URL or "http://host.docker.internal:9000/" + + @property + def minio_root_user(self) -> str: + return self.MINIO_ROOT_USER or "minioadmin" + + @property + def minio_root_password(self) -> str: + return self.MINIO_ROOT_PASSWORD or "minioadmin" + @property def webapp_host(self) -> str: return self.WEBAPP_HOST or "localhost" diff --git a/abdiff/core/__init__.py b/abdiff/core/__init__.py index 4784f45..faf5697 100644 --- a/abdiff/core/__init__.py +++ b/abdiff/core/__init__.py @@ -7,6 +7,7 @@ from abdiff.core.calc_ab_diffs import calc_ab_diffs from abdiff.core.calc_ab_metrics import calc_ab_metrics from abdiff.core.collate_ab_transforms import collate_ab_transforms +from abdiff.core.download_input_files import download_input_files from abdiff.core.init_job import init_job from abdiff.core.init_run import init_run from abdiff.core.run_ab_transforms import run_ab_transforms @@ -15,6 +16,7 @@ "init_job", "init_run", "build_ab_images", + "download_input_files", "run_ab_transforms", "collate_ab_transforms", "calc_ab_diffs", diff --git a/abdiff/core/download_input_files.py b/abdiff/core/download_input_files.py new file mode 100644 index 0000000..1166fcb --- /dev/null +++ b/abdiff/core/download_input_files.py @@ -0,0 +1,85 @@ +import logging +import subprocess + +import boto3 +from botocore.exceptions import ClientError +from mypy_boto3_s3.client import S3Client + +from abdiff.config import Config + +logger = logging.getLogger(__name__) + +CONFIG = Config() + + +def download_input_files(input_files: list[str]) -> None: + """Download extract files from S3 to a local MinIO server. + + For each file download, two AWS CLI commands are run by subprocess. + The output from the first command is piped to the second command. + These commands are further explained below: + + 1. Copy the contents from the input file and direct to stdout. + ``` + aws s3 cp - + ``` + + 2. Given the stdout from the previous command as input, copy the contents + to a similarly named file on the local MinIO server. + ``` + aws s3 cp --endpoint-url --profile minio - + ``` + + Note: An S3 client connected to the local MinIO server will check whether the + file exists prior to any download. + """ + s3_client = boto3.client( + "s3", + endpoint_url=CONFIG.minio_s3_url, + aws_access_key_id=CONFIG.minio_root_user, + aws_secret_access_key=CONFIG.minio_root_password, + ) + + for input_file in input_files: + if check_object_exists(CONFIG.TIMDEX_BUCKET, input_file, s3_client): + logger.info(f"File found for input: {input_file}. Skipping download.") + continue + + logger.info(f"Downloading input file from {CONFIG.TIMDEX_BUCKET}: {input_file}") + copy_command = ["aws", "s3", "cp", input_file, "-"] + upload_command = [ + "aws", + "s3", + "cp", + "--endpoint-url", + CONFIG.minio_s3_url, + "--profile", + "minio", + "-", + input_file, + ] + + try: + copy_process = subprocess.run( + args=copy_command, check=True, capture_output=True + ) + subprocess.run( + args=upload_command, + check=True, + input=copy_process.stdout, + ) + except subprocess.CalledProcessError: + logger.exception(f"Failed to download input file: {input_file}") + + +def check_object_exists(bucket: str, input_file: str, s3_client: S3Client) -> bool: + key = input_file.replace(f"s3://{bucket}/", "") + try: + s3_client.head_object(Bucket=bucket, Key=key) + except ClientError as exception: + if exception.response["Error"]["Code"] == "404": + return False + logger.exception(f"Cannot determine if object exists for key {key}.") + return False + else: + return True diff --git a/abdiff/core/run_ab_transforms.py b/abdiff/core/run_ab_transforms.py index cfd21d8..1e76faf 100644 --- a/abdiff/core/run_ab_transforms.py +++ b/abdiff/core/run_ab_transforms.py @@ -35,6 +35,8 @@ def run_ab_transforms( image_tag_b: str, input_files: list[str], docker_client: docker.client.DockerClient | None = None, + *, + use_local_s3: bool = False, ) -> tuple[list[str], ...]: """Run Docker containers with versioned images of Transmogrifier. @@ -59,6 +61,10 @@ def run_ab_transforms( URIs for input files on S3 are accepted. docker_client (docker.client.DockerClient | None, optional): Docker client. Defaults to None. + use_local_s3 (bool): Boolean indicating whether the container should + access input files from a local MinIO server (i.e., "local S3 bucket") + or from AWS S3. This flag determines the appropriate environment variables + to set for the Docker containers. Default is False. Returns: tuple[list[str], ...]: A tuple containing two lists, where each list contains @@ -95,7 +101,9 @@ def run_ab_transforms( ] # run containers and collect results - futures = run_all_docker_containers(docker_client, input_files, run_configs) + futures = run_all_docker_containers( + docker_client, input_files, run_configs, use_local_s3=use_local_s3 + ) containers, exceptions = collect_container_results(futures) logger.info( f"Successful containers: {len(containers)}, failed containers: {len(exceptions)}" @@ -129,6 +137,8 @@ def run_all_docker_containers( docker_client: docker.client.DockerClient, input_files: list[str], run_configs: list[tuple], + *, + use_local_s3: bool = False, ) -> list[Future]: """Invoke Docker containers to run in parallel via threads. @@ -152,7 +162,11 @@ def run_all_docker_containers( get_transformed_filename(filename_details), docker_client, ) - tasks.append(executor.submit(run_docker_container, *args)) + tasks.append( + executor.submit( + run_docker_container, *args, use_local_s3=use_local_s3 + ) + ) logger.info(f"All {len(tasks)} containers have exited.") return tasks @@ -166,12 +180,27 @@ def run_docker_container( output_file: str, docker_client: docker.client.DockerClient, timeout: int = CONFIG.transmogrifier_timeout, + *, + use_local_s3: bool = False, ) -> tuple[Container, Exception | None]: """Run Transmogrifier via Docker container to transform input file. The container is run in a detached state to capture a container handle for later use but this function waits for the container to exit before returning. """ + if use_local_s3: + environment_variables = { + "AWS_ENDPOINT_URL": CONFIG.minio_s3_container_url, + "AWS_ACCESS_KEY_ID": CONFIG.minio_root_user, + "AWS_SECRET_ACCESS_KEY": CONFIG.minio_root_password, + } + else: + environment_variables = { + "AWS_ACCESS_KEY_ID": CONFIG.AWS_ACCESS_KEY_ID, + "AWS_SECRET_ACCESS_KEY": CONFIG.AWS_SECRET_ACCESS_KEY, + "AWS_SESSION_TOKEN": CONFIG.AWS_SESSION_TOKEN, + } + container = docker_client.containers.run( docker_image, command=[ @@ -180,11 +209,7 @@ def run_docker_container( f"--source={source}", ], detach=True, - environment={ - "AWS_ACCESS_KEY_ID": CONFIG.AWS_ACCESS_KEY_ID, - "AWS_SECRET_ACCESS_KEY": CONFIG.AWS_SECRET_ACCESS_KEY, - "AWS_SESSION_TOKEN": CONFIG.AWS_SESSION_TOKEN, - }, + environment=environment_variables, labels={ "docker_image": docker_image, "source": source, diff --git a/abdiff/helpers/minio/docker-compose.yaml b/abdiff/helpers/minio/docker-compose.yaml new file mode 100644 index 0000000..33385cd --- /dev/null +++ b/abdiff/helpers/minio/docker-compose.yaml @@ -0,0 +1,24 @@ +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2024-10-29T16-01-48Z + command: server --console-address ":9001" /mnt/data + ports: + - "9000:9000" # API port + - "9001:9001" # Console port + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +services: + minio: + <<: *minio-common + volumes: + - ${MINIO_S3_LOCAL_STORAGE}:/mnt/data + + + From 2c5a69190d73c36d694b4a10799a433eeabbafb3 Mon Sep 17 00:00:00 2001 From: jonavellecuerdo Date: Thu, 7 Nov 2024 09:13:58 -0500 Subject: [PATCH 2/2] Address comments * Move download functionality to extras module * Update Makefile command for stopping MinIO server to use .env * Clarify instructions for running local MinIO server in README * Fix flag description for '--download-files' CLI option * Simplify Docker Compose YAML file * Simplify logging in 'download_input_files' * Log clear messages indicating download counts * Log clear messages * Raise RuntimeError if any files fail to download --- Makefile | 9 ++- README.md | 13 ++-- abdiff/cli.py | 7 +- abdiff/config.py | 2 + abdiff/core/__init__.py | 2 +- abdiff/extras/minio/__init__.py | 0 abdiff/extras/minio/docker-compose.yaml | 17 +++++ .../minio}/download_input_files.py | 72 +++++++++++-------- abdiff/helpers/minio/docker-compose.yaml | 24 ------- 9 files changed, 85 insertions(+), 61 deletions(-) create mode 100644 abdiff/extras/minio/__init__.py create mode 100644 abdiff/extras/minio/docker-compose.yaml rename abdiff/{core => extras/minio}/download_input_files.py (56%) delete mode 100644 abdiff/helpers/minio/docker-compose.yaml diff --git a/Makefile b/Makefile index 1c9b1eb..236f658 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) -MINIO_COMPOSE_FILE=abdiff/helpers/minio/docker-compose.yaml +MINIO_COMPOSE_FILE=abdiff/extras/minio/docker-compose.yaml help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ @@ -56,9 +56,12 @@ black-apply: # Apply changes with 'black' ruff-apply: # Resolve 'fixable errors' with 'ruff' pipenv run ruff check --fix . -# Development commands +#################################### +# MinIO local S3 commands +#################################### + start-minio-server: docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) up -d stop-minio-server: - docker compose -f $(MINIO_COMPOSE_FILE) stop \ No newline at end of file + docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) stop \ No newline at end of file diff --git a/README.md b/README.md index 2b8bcbb..dbd4103 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,18 @@ Compare transformed TIMDEX records from two versions (A,B) of Transmogrifier. ### Running a Local MinIO Server -TIMDEX extract files from S3 (i.e., input files to use in transformations) can be downloaded to a local MinIO server hosted with a Docker container. [MinIO is an object storage solution that provides an Amazon Web Services S3-compatible API and supports all core S3 features](https://min.io/docs/minio/kubernetes/upstream/). Downloading extract files improves the runtime of a diff by reducing the number of requests sent to S3 and avoids repeated downloads of extract files. +TIMDEX extract files from S3 (i.e., input files to use in transformations) can be downloaded to a local MinIO server hosted via a Docker container. [MinIO is an object storage solution that provides an Amazon Web Services S3-compatible API and supports all core S3 features](https://min.io/docs/minio/kubernetes/upstream/). The MinIO server acts as a "local S3 file system", allowing the app to access data on disk through an S3 interface. Since the MinIO server runs in a Docker container, it can be easily started when needed and stopped when not in use. Any data stored in the MinIO server will persist as long as the files exist in the directory specified for `MINIO_S3_LOCAL_STORAGE`. + +Downloading extract files improves the runtime of a diff by reducing the number of requests sent to S3 and avoids AWS credentials timing out. Once an extract file is stored in the local MinIO server, the app can access the data from MinIO for all future runs that include the extract file, avoiding repeated downloads of data used across multiple runs. + 1. Configure your `.env` file. In addition to the [required environment variables](#required), the following environment variables must also be set: ```text - MINIO_S3_LOCAL_STORAGE= - TIMDEX_BUCKET= + MINIO_S3_LOCAL_STORAGE=# full file system path to the directory where MinIO stores its object data on the local disk + MINIO_ROOT_USER=# username for root user account for MinIO server + MINIO_ROOT_PASSWORD=# password for root user account MinIO server + TIMDEX_BUCKET=# when using CLI command 'timdex-sources-csv', this is required to know what TIMDEX bucket to use ``` Note: There are additional variables required by the Local MinIO server (see vars prefixed with "MINIO" in [optional environment variables](#optional)). For these variables, defaults are provided in [abdiff.config](abdiff/config.py). @@ -45,7 +50,7 @@ TIMDEX extract files from S3 (i.e., input files to use in transformations) can b 5. Proceed with A/B Diff CLI commands as needed! -Once a diff run is complete, you can stop the local MinIO server using the Makefile command: `make stop-minio-server`. If you're planning to run another diff using the same files -- good news! All you have to do is restart the local MinIO server. Your data will persist as long as the files exist in the directory you specified for `MINIO_S3_LOCAL_STORAGE`. +Once a diff run is complete, you can stop the local MinIO server using the Makefile command: `make stop-minio-server`. If you're planning to run another diff using the same files, all you have to do is restart the local MinIO server. Your data will persist as long as the files exist in the directory you specified for `MINIO_S3_LOCAL_STORAGE`. ## Concepts diff --git a/abdiff/cli.py b/abdiff/cli.py index 77d78fb..df4a7ec 100644 --- a/abdiff/cli.py +++ b/abdiff/cli.py @@ -150,7 +150,12 @@ def init_job( default="Not provided.", ) @click.option( - "--download-files", is_flag=True, help="Pass to skip download of extract files" + "--download-files", + is_flag=True, + help=( + "Pass to download input files from AWS S3 to a local Minio S3 server " + "for Transmogrifier to use." + ), ) def run_diff( job_directory: str, input_files: str, message: str, *, download_files: bool diff --git a/abdiff/config.py b/abdiff/config.py index abd5abe..9f84312 100644 --- a/abdiff/config.py +++ b/abdiff/config.py @@ -32,10 +32,12 @@ def __getattr__(self, name: str) -> Any: # noqa: ANN401 @property def minio_s3_url(self) -> str: + """Host for ABDiff context (host machine) to connect to MinIO.""" return self.MINIO_S3_URL or "http://localhost:9000/" @property def minio_s3_container_url(self) -> str: + """Host for Transmogrifier Docker containers to connect to MinIO.""" return self.MINIO_S3_CONTAINER_URL or "http://host.docker.internal:9000/" @property diff --git a/abdiff/core/__init__.py b/abdiff/core/__init__.py index faf5697..3a018b7 100644 --- a/abdiff/core/__init__.py +++ b/abdiff/core/__init__.py @@ -7,10 +7,10 @@ from abdiff.core.calc_ab_diffs import calc_ab_diffs from abdiff.core.calc_ab_metrics import calc_ab_metrics from abdiff.core.collate_ab_transforms import collate_ab_transforms -from abdiff.core.download_input_files import download_input_files from abdiff.core.init_job import init_job from abdiff.core.init_run import init_run from abdiff.core.run_ab_transforms import run_ab_transforms +from abdiff.extras.minio.download_input_files import download_input_files __all__ = [ "init_job", diff --git a/abdiff/extras/minio/__init__.py b/abdiff/extras/minio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/abdiff/extras/minio/docker-compose.yaml b/abdiff/extras/minio/docker-compose.yaml new file mode 100644 index 0000000..7fc6956 --- /dev/null +++ b/abdiff/extras/minio/docker-compose.yaml @@ -0,0 +1,17 @@ +services: + minio: + image: quay.io/minio/minio:latest + command: server --console-address ":9001" /mnt/data + ports: + - "9000:9000" # API port + - "9001:9001" # Console port + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + volumes: + - ${MINIO_S3_LOCAL_STORAGE}:/mnt/data \ No newline at end of file diff --git a/abdiff/core/download_input_files.py b/abdiff/extras/minio/download_input_files.py similarity index 56% rename from abdiff/core/download_input_files.py rename to abdiff/extras/minio/download_input_files.py index 1166fcb..282621d 100644 --- a/abdiff/core/download_input_files.py +++ b/abdiff/extras/minio/download_input_files.py @@ -40,36 +40,53 @@ def download_input_files(input_files: list[str]) -> None: aws_secret_access_key=CONFIG.minio_root_password, ) - for input_file in input_files: - if check_object_exists(CONFIG.TIMDEX_BUCKET, input_file, s3_client): - logger.info(f"File found for input: {input_file}. Skipping download.") - continue - - logger.info(f"Downloading input file from {CONFIG.TIMDEX_BUCKET}: {input_file}") - copy_command = ["aws", "s3", "cp", input_file, "-"] - upload_command = [ - "aws", - "s3", - "cp", - "--endpoint-url", - CONFIG.minio_s3_url, - "--profile", - "minio", - "-", - input_file, - ] - + success_count = 0 + fail_count = 0 + for i, input_file in enumerate(input_files): try: - copy_process = subprocess.run( - args=copy_command, check=True, capture_output=True - ) - subprocess.run( - args=upload_command, - check=True, - input=copy_process.stdout, + download_input_file(input_file, s3_client) + success_count += 1 + logger.info( + f"Input file: {i + 1} / {len(input_files)}: '{input_file}' " + "available locally for transformation." ) except subprocess.CalledProcessError: - logger.exception(f"Failed to download input file: {input_file}") + fail_count += 1 + logger.info( + f"Input file: {i + 1} / {len(input_files)}: '{input_file}' " + "failed to download." + ) + logger.info( + f"Available input files: {success_count}, missing input files: {fail_count}." + ) + + if fail_count > 0: + raise RuntimeError( # noqa: TRY003 + f"{fail_count} input file(s) failed to download." + ) + + +def download_input_file(input_file: str, s3_client: S3Client) -> None: + if check_object_exists(CONFIG.TIMDEX_BUCKET, input_file, s3_client): + return + copy_command = ["aws", "s3", "cp", input_file, "-"] + upload_command = [ + "aws", + "s3", + "cp", + "--endpoint-url", + CONFIG.minio_s3_url, + "--profile", + "minio", + "-", + input_file, + ] + copy_process = subprocess.run(args=copy_command, check=True, capture_output=True) + subprocess.run( + args=upload_command, + check=True, + input=copy_process.stdout, + ) def check_object_exists(bucket: str, input_file: str, s3_client: S3Client) -> bool: @@ -79,7 +96,6 @@ def check_object_exists(bucket: str, input_file: str, s3_client: S3Client) -> bo except ClientError as exception: if exception.response["Error"]["Code"] == "404": return False - logger.exception(f"Cannot determine if object exists for key {key}.") return False else: return True diff --git a/abdiff/helpers/minio/docker-compose.yaml b/abdiff/helpers/minio/docker-compose.yaml deleted file mode 100644 index 33385cd..0000000 --- a/abdiff/helpers/minio/docker-compose.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Settings and configurations that are common for all containers -x-minio-common: &minio-common - image: quay.io/minio/minio:RELEASE.2024-10-29T16-01-48Z - command: server --console-address ":9001" /mnt/data - ports: - - "9000:9000" # API port - - "9001:9001" # Console port - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 5 - -services: - minio: - <<: *minio-common - volumes: - - ${MINIO_S3_LOCAL_STORAGE}:/mnt/data - - -