diff --git a/Makefile b/Makefile index dc65248..236f658 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ SHELL=/bin/bash DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ) +MINIO_COMPOSE_FILE=abdiff/extras/minio/docker-compose.yaml help: # Preview Makefile commands @awk 'BEGIN { FS = ":.*#"; print "Usage: make \n\nTargets:" } \ @@ -54,3 +55,13 @@ black-apply: # Apply changes with 'black' ruff-apply: # Resolve 'fixable errors' with 'ruff' pipenv run ruff check --fix . + +#################################### +# MinIO local S3 commands +#################################### + +start-minio-server: + docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) up -d + +stop-minio-server: + docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) stop \ No newline at end of file diff --git a/Pipfile b/Pipfile index 8f87bda..8880b2f 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,7 @@ deepdiff = "*" [dev-packages] black = "*" +boto3-stubs = {version = "*", extras = ["s3"]} coveralls = "*" freezegun = "*" ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1ff4aa9..9f0c47d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4811e3bc6c1ebe8265070fcf866a2b88abf2ed5c77182164fdb978c2758bac23" + "sha256": "6742438ee4103c6b5e0b05af917ed22014aca7fc30a75dfce5d27a84fef01483" }, "pipfile-spec": 6, "requires": { @@ -26,20 +26,20 @@ }, "boto3": { "hashes": [ - "sha256:a9c0955df0b52b43749d81bde159343a40ea2a3537a46049336fe8193871b18e", - "sha256:f4124548bb831e13504e805f2fbbfcee06df42fffea0655862c6eb9b95d6d1be" + "sha256:2d5e160b614db55fbee7981001c54476cb827c441cef65b2fcb2c52a62019909", + "sha256:7d9c359bbbc858a60b51c86328db813353c8bd1940212cdbd0a7da835291c2e1" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.35.53" + "version": "==1.35.54" }, "botocore": { "hashes": [ - "sha256:12869640f2f9fab3152ea312a6906d5bc6ae15522cc74b6367ee1c273269a28b", - "sha256:e610ae076ad1eaed5680d3990493659bbabdffd67b15c61d8373a23e4bc41062" + "sha256:131bb59ce59c8a939b31e8e647242d70cf11d32d4529fa4dca01feea1e891a76", + "sha256:9cca1811094b6cdc144c2c063a3ec2db6d7c88194b04d4277cd34fc8e3473aff" ], "markers": "python_version >= '3.8'", - "version": "==1.35.53" + "version": "==1.35.54" }, "certifi": { "hashes": [ @@ -262,62 +262,62 @@ }, "duckdb": { "hashes": [ - "sha256:0107de622fe208142a1108263a03c43956048dcc99be3702d8e5d2aeaf99554c", - "sha256:099d99dd48d6e4682a3dd6233ceab73d977ebe1a87afaac54cf77c844e24514a", - "sha256:181edb1973bd8f493bcb6ecfa035f1a592dff4667758592f300619012ba251c0", - "sha256:19142a77e72874aeaa6fda30aeb13612c6de5e8c60fbcc3392cea6ef0694eeaf", - "sha256:19386aa09f0d6f97634ba2972096d1c80d880176dfb0e949eadc91c98262a663", - "sha256:1bba58459ad897a78c4e478a097626fc266459a40338cecc68a49a8d5dc72fb7", - "sha256:25889e6e29b87047b1dd56385ac08156e4713c59326cc6fff89657d01b2c417b", - "sha256:312570fa5277c3079de18388b86c2d87cbe1044838bb152b235c0227581d5d42", - "sha256:492b1d86a696428bd3f14dc1c7c3230e2dbca8978f288be64b04a26e0e00fad5", - "sha256:4ce949f1d7999aa6a046eb64067eee41d4c5c2872ba4fa408c9947742d0c7231", - "sha256:564166811c68d9c7f9911eb707ad32ec9c2507b98336d894fbe658b85bf1c697", - "sha256:568439ea4fce8cb72ec1f767cd510686a9e7e29a011fc7c56d990059a6e94e48", - "sha256:578e0953e4d8ba8da0cd69fb2930c45f51ce47d213b77d8a4cd461f9c0960b87", - "sha256:623cb1952466aae5907af84107bcdec25a5ca021a8b6441e961f41edc724f6f2", - "sha256:6858e10c60ff7e70e61d3dd53d2545c8b2609942e45fd6de38cd0dee52932de3", - "sha256:695dcbc561374b126e86659709feadf883c9969ed718e94713edd4ba15d16619", - "sha256:6c37b039f6d6fed14d89450f5ccf54922b3304192d7412e12d6cc8d9e757f7a2", - "sha256:6ca722738fa9eb6218619740631de29acfdd132de6f6a6350fee5e291c2f6117", - "sha256:7111fd3e7b334a7be383313ce29918b7c643e4f6ef44d6d63c3ab3fa6716c114", - "sha256:72b5eb5762c1a5e68849c7143f3b3747a9f15c040e34e41559f233a1569ad16f", - "sha256:74974f2d7210623a5d61b1fb0cb589c6e5ffcbf7dbb757a04c5ba24adcfc8cac", - "sha256:7ca967c5a57b1d0cb0fd5e539ab24110e5a59dcbedd365bb2dc80533d6e44a8d", - "sha256:83372b1b411086cac01ab2071122772fa66170b1b41ddbc37527464066083668", - "sha256:84103373e818758dfa361d27781d0f096553843c5ffb9193260a0786c5248270", - "sha256:87e972bd452eeeab197fe39dcaeecdb7c264b1f75a0ee67e532e235fe45b84df", - "sha256:89164a2d29d56605a95ee5032aa415dd487028c4fd3e06d971497840e74c56e7", - "sha256:8a01fae9604a54ecbc26e7503c522311f15afbd2870e6d8f6fbef4545dfae550", - "sha256:8a09610f780857677725897856f8cdf3cafd8a991f871e6cb8ba88b2dbc8d737", - "sha256:91e7f99cf5cab1d26f92cb014429153497d805e79689baa44f4c4585a8cb243f", - "sha256:9283dcca87c3260eb631a99d738fa72b8545ed45b475bc72ad254f7310e14284", - "sha256:9a6b73e70b73c8df85da383f6e557c03cad5c877868b9a7e41715761e8166c1e", - "sha256:9b4c6b6a08180261d98330d97355503961a25ca31cd9ef296e0681f7895b4a2c", - "sha256:9ba6d1f918e6ca47a368a0c32806016405cb9beb2c245806b0ca998f569d2bdf", - "sha256:ada29be1e889f486c6cf1f6dffd15463e748faf361f33996f2e862779edc24a9", - "sha256:be86e586ca7af7e807f72479a2b8d0983565360b19dbda4ef8a9d7b3909b8e2c", - "sha256:bfdfd23e2bf58014ad0673973bd0ed88cd048dfe8e82420814a71d7d52ef2288", - "sha256:c0f0ddac0482f0f3fece54d720d13819e82ae26c01a939ffa66a87be53f7f665", - "sha256:c796d33f1e5a0c8c570d22da0c0b1db8578687e427029e1ce2c8ce3f9fffa6a3", - "sha256:c8232861dc8ec6daa29067056d5a0e5789919f2ab22ab792787616d7cd52f02a", - "sha256:d395a3bf510bf24686821eec15802624797dcb33e8f14f8a7cc8e17d909474af", - "sha256:d8caaf43909e49537e26df51d80d075ae2b25a610d28ed8bd31d6ccebeaf3c65", - "sha256:d9fc0b550f96901fa7e76dc70a13f6477ad3e18ef1cb21d414c3a5569de3f27e", - "sha256:db37441deddfee6ac35a0c742d2f9e90e4e50b9e76d586a060d122b8fc56dada", - "sha256:e26422a3358c816d764639070945b73eef55d1b4df990989e3492c85ef725c21", - "sha256:e3e6300b7ccaf64b609f4f0780a6e1d25ab8cf34cceed46e62c35b6c4c5cb63b", - "sha256:e8c766b87f675c76d6d17103bf6fb9fb1a9e2fcb3d9b25c28bbc634bde31223e", - "sha256:e9e8387bcc9a591ad14011ddfec0d408d1d9b1889c6c9b495a04c7016a24b9b3", - "sha256:efec169b3fe0b821e3207ba3e445f227d42dd62b4440ff79c37fa168a4fc5a71", - "sha256:f5c0996988a70dd3bc8111d9b9aeab7e38ed1999a52607c5f1b528e362b4dd1c", - "sha256:f87edaf20001530e63a4f7bda13b55dc3152d7171226915f2bf34e0813c8759e", - "sha256:f8c5ff4970403ed3ff0ac71fe0ce1e6be3199df9d542afc84c424b444ba4ffe8", - "sha256:fd800f75728727fe699ed1eb22b636867cf48c9dd105ee88b977e20c89df4509" + "sha256:00cca22df96aa3473fe4584f84888e2cf1c516e8c2dd837210daec44eadba586", + "sha256:08935700e49c187fe0e9b2b86b5aad8a2ccd661069053e38bfaed3b9ff795efd", + "sha256:0897f83c09356206ce462f62157ce064961a5348e31ccb2a557a7531d814e70e", + "sha256:09c68522c30fc38fc972b8a75e9201616b96ae6da3444585f14cf0d116008c95", + "sha256:0a55169d2d2e2e88077d91d4875104b58de45eff6a17a59c7dc41562c73df4be", + "sha256:0ba6baa0af33ded836b388b09433a69b8bec00263247f6bf0a05c65c897108d3", + "sha256:183ac743f21c6a4d6adfd02b69013d5fd78e5e2cd2b4db023bc8a95457d4bc5d", + "sha256:1aa3abec8e8995a03ff1a904b0e66282d19919f562dd0a1de02f23169eeec461", + "sha256:1c0226dc43e2ee4cc3a5a4672fddb2d76fd2cf2694443f395c02dd1bea0b7fce", + "sha256:1d9ab6143e73bcf17d62566e368c23f28aa544feddfd2d8eb50ef21034286f24", + "sha256:2141c6b28162199999075d6031b5d63efeb97c1e68fb3d797279d31c65676269", + "sha256:252d9b17d354beb9057098d4e5d5698e091a4f4a0d38157daeea5fc0ec161670", + "sha256:25fb02629418c0d4d94a2bc1776edaa33f6f6ccaa00bd84eb96ecb97ae4b50e9", + "sha256:2f073d15d11a328f2e6d5964a704517e818e930800b7f3fa83adea47f23720d3", + "sha256:35c420f58abc79a68a286a20fd6265636175fadeca1ce964fc8ef159f3acc289", + "sha256:4ebf5f60ddbd65c13e77cddb85fe4af671d31b851f125a4d002a313696af43f1", + "sha256:4f0e2e5a6f5a53b79aee20856c027046fba1d73ada6178ed8467f53c3877d5e0", + "sha256:51c6d79e05b4a0933672b1cacd6338f882158f45ef9903aef350c4427d9fc898", + "sha256:51e7dbd968b393343b226ab3f3a7b5a68dee6d3fe59be9d802383bf916775cb8", + "sha256:5ace6e4b1873afdd38bd6cc8fcf90310fb2d454f29c39a61d0c0cf1a24ad6c8d", + "sha256:5d57776539211e79b11e94f2f6d63de77885f23f14982e0fac066f2885fcf3ff", + "sha256:6411e21a2128d478efbd023f2bdff12464d146f92bc3e9c49247240448ace5a6", + "sha256:647f17bd126170d96a38a9a6f25fca47ebb0261e5e44881e3782989033c94686", + "sha256:68c3a46ab08836fe041d15dcbf838f74a990d551db47cb24ab1c4576fc19351c", + "sha256:77f26884c7b807c7edd07f95cf0b00e6d47f0de4a534ac1706a58f8bc70d0d31", + "sha256:7c71169fa804c0b65e49afe423ddc2dc83e198640e3b041028da8110f7cd16f7", + "sha256:80158f4c7c7ada46245837d5b6869a336bbaa28436fbb0537663fa324a2750cd", + "sha256:872d38b65b66e3219d2400c732585c5b4d11b13d7a36cd97908d7981526e9898", + "sha256:8ee97ec337794c162c0638dda3b4a30a483d0587deda22d45e1909036ff0b739", + "sha256:911d58c22645bfca4a5a049ff53a0afd1537bc18fedb13bc440b2e5af3c46148", + "sha256:9c619e4849837c8c83666f2cd5c6c031300cd2601e9564b47aa5de458ff6e69d", + "sha256:9d0767ada9f06faa5afcf63eb7ba1befaccfbcfdac5ff86f0168c673dd1f47aa", + "sha256:9e3f5cd604e7c39527e6060f430769b72234345baaa0987f9500988b2814f5e4", + "sha256:a1f83c7217c188b7ab42e6a0963f42070d9aed114f6200e3c923c8899c090f16", + "sha256:a1fa0c502f257fa9caca60b8b1478ec0f3295f34bb2efdc10776fc731b8a6c5f", + "sha256:a30dd599b8090ea6eafdfb5a9f1b872d78bac318b6914ada2d35c7974d643640", + "sha256:a433ae9e72c5f397c44abdaa3c781d94f94f4065bcbf99ecd39433058c64cb38", + "sha256:a4748635875fc3c19a7320a6ae7410f9295557450c0ebab6d6712de12640929a", + "sha256:b74e121ab65dbec5290f33ca92301e3a4e81797966c8d9feef6efdf05fc6dafd", + "sha256:c443d3d502335e69fc1e35295fcfd1108f72cb984af54c536adfd7875e79cee5", + "sha256:c5336939d83837af52731e02b6a78a446794078590aa71fd400eb17f083dda3e", + "sha256:cddc6c1a3b91dcc5f32493231b3ba98f51e6d3a44fe02839556db2b928087378", + "sha256:d08308e0a46c748d9c30f1d67ee1143e9c5ea3fbcccc27a47e115b19e7e78aa9", + "sha256:d5724fd8a49e24d730be34846b814b98ba7c304ca904fbdc98b47fa95c0b0cee", + "sha256:e4ef7ba97a65bd39d66f2a7080e6fb60e7c3e41d4c1e19245f90f53b98e3ac32", + "sha256:e59087dbbb63705f2483544e01cccf07d5b35afa58be8931b224f3221361d537", + "sha256:e86006958e84c5c02f08f9b96f4bc26990514eab329b1b4f71049b3727ce5989", + "sha256:ecb1dc9062c1cc4d2d88a5e5cd8cc72af7818ab5a3c0f796ef0ffd60cfd3efb4", + "sha256:eeacb598120040e9591f5a4edecad7080853aa8ac27e62d280f151f8c862afa3", + "sha256:f549af9f7416573ee48db1cf8c9d27aeed245cb015f4b4f975289418c6cf7320", + "sha256:f58db1b65593ff796c8ea6e63e2e144c944dd3d51c8d8e40dffa7f41693d35d3", + "sha256:f9b47036945e1db32d70e414a10b1593aec641bd4c5e2056873d971cc21e978b" ], "index": "pypi", "markers": "python_full_version >= '3.7.0'", - "version": "==1.1.2" + "version": "==1.1.3" }, "flask": { "hashes": [ @@ -529,62 +529,64 @@ }, "numpy": { "hashes": [ - "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8", - "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466", - "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35", - "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c", - "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4", - "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6", - "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0", - "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7", - "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a", - "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a", - "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e", - "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62", - "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2", - "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5", - "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee", - "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe", - "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a", - "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e", - "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf", - "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c", - "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3", - "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86", - "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df", - "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98", - "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d", - "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2", - "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146", - "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550", - "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8", - "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb", - "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e", - "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d", - "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366", - "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0", - "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db", - "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe", - "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426", - "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952", - "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03", - "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f", - "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7", - "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b", - "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17", - "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5", - "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1", - "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142", - "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884", - "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a", - "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9", - "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445", - "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1", - "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1", - "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648" + "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe", + "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0", + "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48", + "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a", + "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564", + "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958", + "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17", + "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0", + "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee", + "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b", + "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4", + "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4", + "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6", + "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4", + "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d", + "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f", + "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f", + "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f", + "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56", + "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9", + "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd", + "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23", + "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed", + "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a", + "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098", + "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1", + "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512", + "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f", + "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09", + "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f", + "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc", + "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8", + "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0", + "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761", + "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef", + "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5", + "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e", + "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b", + "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d", + "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43", + "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c", + "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41", + "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff", + "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408", + "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2", + "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9", + "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57", + "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb", + "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9", + "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3", + "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a", + "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0", + "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e", + "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598", + "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4" ], "markers": "python_version >= '3.12'", - "version": "==2.1.2" + "version": "==2.1.3" }, "orderly-set": { "hashes": [ @@ -741,7 +743,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pytz": { @@ -772,7 +774,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "types-cffi": { @@ -811,11 +813,11 @@ }, "types-setuptools": { "hashes": [ - "sha256:2949913a518d5285ce00a3b7d88961c80a6e72ffb8f3da0a3f5650ea533bd45e", - "sha256:6721ac0f1a620321e2ccd87a9a747c4a383dc381f78d894ce37f2455b45fcf1c" + "sha256:080883032fb26dfb91c748f69e4efbb202aeeb3f9e95e68bf7352b7ab215069f", + "sha256:c5a98b3308fea29d26228978bd83f6ea262d2fd4d967e34e8e49d04708bb3022" ], "markers": "python_version >= '3.8'", - "version": "==75.2.0.20241025" + "version": "==75.3.0.20241105" }, "tzdata": { "hashes": [ @@ -835,11 +837,11 @@ }, "werkzeug": { "hashes": [ - "sha256:8cd39dfbdfc1e051965f156163e2974e52c210f130810e9ad36858f0fd3edad4", - "sha256:a71124d1ef06008baafa3d266c02f56e1836a5984afd6dd6c9230669d60d9fb5" + "sha256:4f7d1a5de312c810a8a2c6f0b47e9f6a7cffb7c8322def35e4d4d9841ff85597", + "sha256:f471a4cd167233077e9d2a8190c3471c5bc520c636a9e3c1e9300c33bced03bc" ], "markers": "python_version >= '3.9'", - "version": "==3.1.1" + "version": "==3.1.2" } }, "develop": { @@ -879,6 +881,25 @@ "markers": "python_version >= '3.9'", "version": "==24.10.0" }, + "boto3-stubs": { + "extras": [ + "s3" + ], + "hashes": [ + "sha256:17519739590614d50a4bfb82d10e02a9bf324ab87e90d54ecddf68044ba219b7", + "sha256:ecf6cce737abc9bc212d48238021ff4aa3fb20ed887ecdee72a750bd329c902e" + ], + "markers": "python_version >= '3.8'", + "version": "==1.35.54" + }, + "botocore-stubs": { + "hashes": [ + "sha256:26ba65907eed959dddc644ab1cd72e3a2cc9761dad79e0b45ff3b8676c47e5ec", + "sha256:49e28813324308bfc5a92bde118df5c9c41a01237eef1e1628891770f3f68a94" + ], + "markers": "python_version >= '3.8'", + "version": "==1.35.54" + }, "certifi": { "hashes": [ "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", @@ -1229,6 +1250,13 @@ "markers": "python_version >= '3.8'", "version": "==1.13.0" }, + "mypy-boto3-s3": { + "hashes": [ + "sha256:34d19dfba400f5b9bd6b64f09eb8f8eedef60545b410a3753fe99fec0c41ba78", + "sha256:f0087a3765d103b2db565cd8065ebc2b0f70f2dd4e92c132f64b8945dd869940" + ], + "version": "==1.35.46" + }, "mypy-extensions": { "hashes": [ "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", @@ -1247,62 +1275,64 @@ }, "numpy": { "hashes": [ - "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8", - "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466", - "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35", - "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c", - "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4", - "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6", - "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0", - "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7", - "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a", - "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a", - "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e", - "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62", - "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2", - "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5", - "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee", - "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe", - "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a", - "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e", - "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf", - "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c", - "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3", - "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86", - "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df", - "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98", - "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d", - "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2", - "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146", - "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550", - "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8", - "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb", - "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e", - "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d", - "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366", - "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0", - "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db", - "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe", - "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426", - "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952", - "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03", - "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f", - "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7", - "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b", - "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17", - "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5", - "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1", - "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142", - "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884", - "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a", - "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9", - "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445", - "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1", - "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1", - "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648" + "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe", + "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0", + "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48", + "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a", + "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564", + "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958", + "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17", + "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0", + "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee", + "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b", + "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4", + "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4", + "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6", + "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4", + "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d", + "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f", + "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f", + "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f", + "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56", + "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9", + "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd", + "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23", + "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed", + "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a", + "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098", + "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1", + "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512", + "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f", + "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09", + "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f", + "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc", + "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8", + "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0", + "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761", + "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef", + "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5", + "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e", + "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b", + "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d", + "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43", + "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c", + "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41", + "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff", + "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408", + "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2", + "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9", + "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57", + "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb", + "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9", + "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3", + "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a", + "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0", + "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e", + "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598", + "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4" ], "markers": "python_version >= '3.12'", - "version": "==2.1.2" + "version": "==2.1.3" }, "packaging": { "hashes": [ @@ -1472,7 +1502,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pyyaml": { @@ -1581,7 +1611,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "stack-data": { @@ -1599,6 +1629,14 @@ "markers": "python_version >= '3.8'", "version": "==5.14.3" }, + "types-awscrt": { + "hashes": [ + "sha256:3fd1edeac923d1956c0e907c973fb83bda465beae7f054716b371b293f9b5fdc", + "sha256:517d9d06f19cf58d778ca90ad01e52e0489466bf70dcf78c7f47f74fdf151a60" + ], + "markers": "python_version >= '3.8'", + "version": "==0.23.0" + }, "types-pytz": { "hashes": [ "sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7", @@ -1607,6 +1645,14 @@ "markers": "python_version >= '3.8'", "version": "==2024.2.0.20241003" }, + "types-s3transfer": { + "hashes": [ + "sha256:d34c5a82f531af95bb550927136ff5b737a1ed3087f90a59d545591dfde5b4cc", + "sha256:f761b2876ac4c208e6c6b75cdf5f6939009768be9950c545b11b0225e7703ee7" + ], + "markers": "python_version >= '3.8'", + "version": "==0.10.3" + }, "typing-extensions": { "hashes": [ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", diff --git a/README.md b/README.md index c97e327..dbd4103 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,43 @@ Compare transformed TIMDEX records from two versions (A,B) of Transmogrifier. - To lint the repo: `make lint` - To run the app: `pipenv run abdiff --help` +### Running a Local MinIO Server + +TIMDEX extract files from S3 (i.e., input files to use in transformations) can be downloaded to a local MinIO server hosted via a Docker container. [MinIO is an object storage solution that provides an Amazon Web Services S3-compatible API and supports all core S3 features](https://min.io/docs/minio/kubernetes/upstream/). The MinIO server acts as a "local S3 file system", allowing the app to access data on disk through an S3 interface. Since the MinIO server runs in a Docker container, it can be easily started when needed and stopped when not in use. Any data stored in the MinIO server will persist as long as the files exist in the directory specified for `MINIO_S3_LOCAL_STORAGE`. + +Downloading extract files improves the runtime of a diff by reducing the number of requests sent to S3 and avoids AWS credentials timing out. Once an extract file is stored in the local MinIO server, the app can access the data from MinIO for all future runs that include the extract file, avoiding repeated downloads of data used across multiple runs. + + +1. Configure your `.env` file. In addition to the [required environment variables](#required), the following environment variables must also be set: + + ```text + MINIO_S3_LOCAL_STORAGE=# full file system path to the directory where MinIO stores its object data on the local disk + MINIO_ROOT_USER=# username for root user account for MinIO server + MINIO_ROOT_PASSWORD=# password for root user account MinIO server + TIMDEX_BUCKET=# when using CLI command 'timdex-sources-csv', this is required to know what TIMDEX bucket to use + ``` + + Note: There are additional variables required by the Local MinIO server (see vars prefixed with "MINIO" in [optional environment variables](#optional)). For these variables, defaults are provided in [abdiff.config](abdiff/config.py). + +2. Create an AWS profile `minio`. When prompted for an "AWS Access Key ID" and "AWS Secret Access Key", pass the values set for the `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD` environment variables, respectively. + ```shell + aws configure --profile minio + ``` + +3. Launch a local MinIO server via Docker container by running the Makefile command: + ```shell + make start-minio-server + ``` + + The API is accessible at: http://127.0.0.1:9000. + The WebUI is accessible at: http://127.0.0.1:9001. + +4. On your browser, navigate to the WebUI and sign into the local MinIO server. Create a bucket in the local MinIO server named after the S3 bucket containing the TIMDEX extract files that will be used in the A/B Diff. + +5. Proceed with A/B Diff CLI commands as needed! + +Once a diff run is complete, you can stop the local MinIO server using the Makefile command: `make stop-minio-server`. If you're planning to run another diff using the same files, all you have to do is restart the local MinIO server. Your data will persist as long as the files exist in the directory you specified for `MINIO_S3_LOCAL_STORAGE`. + ## Concepts A **Job** in `abdiff` represents the A/B test for comparing the results from two versions of Transmogrifier. When a job is first created, a working directory and a JSON file `job.json` with an initial set of configurations is created. @@ -90,6 +127,11 @@ AWS_SESSION_TOKEN=# passed to Transmogrifier containers for use ### Optional ```text +MINIO_S3_LOCAL_STORAGE=# full file system path to the directory where MinIO stores its object data on the local disk +MINIO_S3_URL=# endpoint for MinIO server API; default is "http://localhost:9000/" +MINIO_S3_CONTAINER_URL=# endpoint for the MinIO server when acccessed from inside a Docker container; default is "http://host.docker.internal:9000/" +MINIO_ROOT_USER=# username for root user account for MinIO server +MINIO_ROOT_PASSWORD=# password for root user account MinIO server WEBAPP_HOST=# host for flask webapp WEBAPP_PORT=# port for flask webapp TRANSMOGRIFIER_MAX_WORKERS=# max number of Transmogrifier containers to run in parallel; default is 6 diff --git a/abdiff/cli.py b/abdiff/cli.py index c42aaef..df4a7ec 100644 --- a/abdiff/cli.py +++ b/abdiff/cli.py @@ -14,6 +14,7 @@ calc_ab_diffs, calc_ab_metrics, collate_ab_transforms, + download_input_files, init_run, run_ab_transforms, ) @@ -148,7 +149,17 @@ def init_job( help="Message to describe Run.", default="Not provided.", ) -def run_diff(job_directory: str, input_files: str, message: str) -> None: +@click.option( + "--download-files", + is_flag=True, + help=( + "Pass to download input files from AWS S3 to a local Minio S3 server " + "for Transmogrifier to use." + ), +) +def run_diff( + job_directory: str, input_files: str, message: str, *, download_files: bool +) -> None: job_data = read_job_json(job_directory) run_directory = init_run(job_directory, message=message) @@ -160,11 +171,15 @@ def run_diff(job_directory: str, input_files: str, message: str) -> None: else: input_files_list = [filepath.strip() for filepath in input_files.split(",")] + if download_files: + download_input_files(input_files_list) + ab_transformed_file_lists = run_ab_transforms( run_directory=run_directory, image_tag_a=job_data["image_tag_a"], image_tag_b=job_data["image_tag_b"], input_files=input_files_list, + use_local_s3=download_files, ) collated_dataset_path = collate_ab_transforms( run_directory=run_directory, diff --git a/abdiff/config.py b/abdiff/config.py index 66eaee1..9f84312 100644 --- a/abdiff/config.py +++ b/abdiff/config.py @@ -11,6 +11,11 @@ class Config: "WORKSPACE", ) OPTIONAL_ENV_VARS = ( + "MINIO_S3_LOCAL_STORAGE", + "MINIO_S3_URL", + "MINIO_S3_CONTAINER_URL", + "MINIO_ROOT_USER", + "MINIO_ROOT_PASSWORD", "WEBAPP_HOST", "WEBAPP_PORT", "TRANSMOGRIFIER_MAX_WORKERS", @@ -25,6 +30,24 @@ def __getattr__(self, name: str) -> Any: # noqa: ANN401 message = f"'{name}' not a valid configuration variable" raise AttributeError(message) + @property + def minio_s3_url(self) -> str: + """Host for ABDiff context (host machine) to connect to MinIO.""" + return self.MINIO_S3_URL or "http://localhost:9000/" + + @property + def minio_s3_container_url(self) -> str: + """Host for Transmogrifier Docker containers to connect to MinIO.""" + return self.MINIO_S3_CONTAINER_URL or "http://host.docker.internal:9000/" + + @property + def minio_root_user(self) -> str: + return self.MINIO_ROOT_USER or "minioadmin" + + @property + def minio_root_password(self) -> str: + return self.MINIO_ROOT_PASSWORD or "minioadmin" + @property def webapp_host(self) -> str: return self.WEBAPP_HOST or "localhost" diff --git a/abdiff/core/__init__.py b/abdiff/core/__init__.py index 4784f45..3a018b7 100644 --- a/abdiff/core/__init__.py +++ b/abdiff/core/__init__.py @@ -10,11 +10,13 @@ from abdiff.core.init_job import init_job from abdiff.core.init_run import init_run from abdiff.core.run_ab_transforms import run_ab_transforms +from abdiff.extras.minio.download_input_files import download_input_files __all__ = [ "init_job", "init_run", "build_ab_images", + "download_input_files", "run_ab_transforms", "collate_ab_transforms", "calc_ab_diffs", diff --git a/abdiff/core/run_ab_transforms.py b/abdiff/core/run_ab_transforms.py index cfd21d8..1e76faf 100644 --- a/abdiff/core/run_ab_transforms.py +++ b/abdiff/core/run_ab_transforms.py @@ -35,6 +35,8 @@ def run_ab_transforms( image_tag_b: str, input_files: list[str], docker_client: docker.client.DockerClient | None = None, + *, + use_local_s3: bool = False, ) -> tuple[list[str], ...]: """Run Docker containers with versioned images of Transmogrifier. @@ -59,6 +61,10 @@ def run_ab_transforms( URIs for input files on S3 are accepted. docker_client (docker.client.DockerClient | None, optional): Docker client. Defaults to None. + use_local_s3 (bool): Boolean indicating whether the container should + access input files from a local MinIO server (i.e., "local S3 bucket") + or from AWS S3. This flag determines the appropriate environment variables + to set for the Docker containers. Default is False. Returns: tuple[list[str], ...]: A tuple containing two lists, where each list contains @@ -95,7 +101,9 @@ def run_ab_transforms( ] # run containers and collect results - futures = run_all_docker_containers(docker_client, input_files, run_configs) + futures = run_all_docker_containers( + docker_client, input_files, run_configs, use_local_s3=use_local_s3 + ) containers, exceptions = collect_container_results(futures) logger.info( f"Successful containers: {len(containers)}, failed containers: {len(exceptions)}" @@ -129,6 +137,8 @@ def run_all_docker_containers( docker_client: docker.client.DockerClient, input_files: list[str], run_configs: list[tuple], + *, + use_local_s3: bool = False, ) -> list[Future]: """Invoke Docker containers to run in parallel via threads. @@ -152,7 +162,11 @@ def run_all_docker_containers( get_transformed_filename(filename_details), docker_client, ) - tasks.append(executor.submit(run_docker_container, *args)) + tasks.append( + executor.submit( + run_docker_container, *args, use_local_s3=use_local_s3 + ) + ) logger.info(f"All {len(tasks)} containers have exited.") return tasks @@ -166,12 +180,27 @@ def run_docker_container( output_file: str, docker_client: docker.client.DockerClient, timeout: int = CONFIG.transmogrifier_timeout, + *, + use_local_s3: bool = False, ) -> tuple[Container, Exception | None]: """Run Transmogrifier via Docker container to transform input file. The container is run in a detached state to capture a container handle for later use but this function waits for the container to exit before returning. """ + if use_local_s3: + environment_variables = { + "AWS_ENDPOINT_URL": CONFIG.minio_s3_container_url, + "AWS_ACCESS_KEY_ID": CONFIG.minio_root_user, + "AWS_SECRET_ACCESS_KEY": CONFIG.minio_root_password, + } + else: + environment_variables = { + "AWS_ACCESS_KEY_ID": CONFIG.AWS_ACCESS_KEY_ID, + "AWS_SECRET_ACCESS_KEY": CONFIG.AWS_SECRET_ACCESS_KEY, + "AWS_SESSION_TOKEN": CONFIG.AWS_SESSION_TOKEN, + } + container = docker_client.containers.run( docker_image, command=[ @@ -180,11 +209,7 @@ def run_docker_container( f"--source={source}", ], detach=True, - environment={ - "AWS_ACCESS_KEY_ID": CONFIG.AWS_ACCESS_KEY_ID, - "AWS_SECRET_ACCESS_KEY": CONFIG.AWS_SECRET_ACCESS_KEY, - "AWS_SESSION_TOKEN": CONFIG.AWS_SESSION_TOKEN, - }, + environment=environment_variables, labels={ "docker_image": docker_image, "source": source, diff --git a/abdiff/extras/minio/__init__.py b/abdiff/extras/minio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/abdiff/extras/minio/docker-compose.yaml b/abdiff/extras/minio/docker-compose.yaml new file mode 100644 index 0000000..7fc6956 --- /dev/null +++ b/abdiff/extras/minio/docker-compose.yaml @@ -0,0 +1,17 @@ +services: + minio: + image: quay.io/minio/minio:latest + command: server --console-address ":9001" /mnt/data + ports: + - "9000:9000" # API port + - "9001:9001" # Console port + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + volumes: + - ${MINIO_S3_LOCAL_STORAGE}:/mnt/data \ No newline at end of file diff --git a/abdiff/extras/minio/download_input_files.py b/abdiff/extras/minio/download_input_files.py new file mode 100644 index 0000000..282621d --- /dev/null +++ b/abdiff/extras/minio/download_input_files.py @@ -0,0 +1,101 @@ +import logging +import subprocess + +import boto3 +from botocore.exceptions import ClientError +from mypy_boto3_s3.client import S3Client + +from abdiff.config import Config + +logger = logging.getLogger(__name__) + +CONFIG = Config() + + +def download_input_files(input_files: list[str]) -> None: + """Download extract files from S3 to a local MinIO server. + + For each file download, two AWS CLI commands are run by subprocess. + The output from the first command is piped to the second command. + These commands are further explained below: + + 1. Copy the contents from the input file and direct to stdout. + ``` + aws s3 cp - + ``` + + 2. Given the stdout from the previous command as input, copy the contents + to a similarly named file on the local MinIO server. + ``` + aws s3 cp --endpoint-url --profile minio - + ``` + + Note: An S3 client connected to the local MinIO server will check whether the + file exists prior to any download. + """ + s3_client = boto3.client( + "s3", + endpoint_url=CONFIG.minio_s3_url, + aws_access_key_id=CONFIG.minio_root_user, + aws_secret_access_key=CONFIG.minio_root_password, + ) + + success_count = 0 + fail_count = 0 + for i, input_file in enumerate(input_files): + try: + download_input_file(input_file, s3_client) + success_count += 1 + logger.info( + f"Input file: {i + 1} / {len(input_files)}: '{input_file}' " + "available locally for transformation." + ) + except subprocess.CalledProcessError: + fail_count += 1 + logger.info( + f"Input file: {i + 1} / {len(input_files)}: '{input_file}' " + "failed to download." + ) + logger.info( + f"Available input files: {success_count}, missing input files: {fail_count}." + ) + + if fail_count > 0: + raise RuntimeError( # noqa: TRY003 + f"{fail_count} input file(s) failed to download." + ) + + +def download_input_file(input_file: str, s3_client: S3Client) -> None: + if check_object_exists(CONFIG.TIMDEX_BUCKET, input_file, s3_client): + return + copy_command = ["aws", "s3", "cp", input_file, "-"] + upload_command = [ + "aws", + "s3", + "cp", + "--endpoint-url", + CONFIG.minio_s3_url, + "--profile", + "minio", + "-", + input_file, + ] + copy_process = subprocess.run(args=copy_command, check=True, capture_output=True) + subprocess.run( + args=upload_command, + check=True, + input=copy_process.stdout, + ) + + +def check_object_exists(bucket: str, input_file: str, s3_client: S3Client) -> bool: + key = input_file.replace(f"s3://{bucket}/", "") + try: + s3_client.head_object(Bucket=bucket, Key=key) + except ClientError as exception: + if exception.response["Error"]["Code"] == "404": + return False + return False + else: + return True