From 733e578dd19c6f9edc39f594f60c50ea34ad61c2 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 28 Jul 2023 22:29:52 +0200 Subject: [PATCH 01/13] Moved tqdm to long(est) running process --- libretro_finder/main.py | 5 ++--- libretro_finder/utils.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libretro_finder/main.py b/libretro_finder/main.py index dd6c492..cb7d28f 100644 --- a/libretro_finder/main.py +++ b/libretro_finder/main.py @@ -2,7 +2,6 @@ import shutil import pathlib import numpy as np -import tqdm from config import SYSTEMS as system_df from libretro_finder.utils import match_arrays, recursive_hash @@ -49,7 +48,7 @@ def organize( # printing matches per system matches = system_subset.groupby("system").count() print( - f"{matches['name'].sum()} matching BIOS files were found for {matches.shape[0]}" + f"{matches['name'].sum()} matching BIOS files were found for {matches.shape[0]} " "unique systems:" ) for name, row in matches.iterrows(): @@ -61,7 +60,7 @@ def organize( # checking whether our input and output paths are of equal length assert len(srcs) == len(dsts) - for i in tqdm.tqdm(range(srcs.size), total=srcs.size): + for i in range(srcs.size): dst = output_dir / dsts[i] parent = dst.parent if dst.exists() or srcs[i] == dst: diff --git a/libretro_finder/utils.py b/libretro_finder/utils.py index 7f4b44d..7117995 100644 --- a/libretro_finder/utils.py +++ b/libretro_finder/utils.py @@ -2,7 +2,7 @@ import hashlib import pathlib from typing import Tuple - +from tqdm import tqdm import numpy as np from config import MAX_BIOS_BYTES @@ -34,12 +34,18 @@ def recursive_hash( file_paths = [ file_path for file_path in file_paths - if file_path.stat().st_size <= MAX_BIOS_BYTES and file_path.is_file() + if file_path.exists() + and file_path.stat().st_size <= MAX_BIOS_BYTES + and file_path.is_file() ] file_hashes = [] with concurrent.futures.ThreadPoolExecutor() as executor: - for file_hash in executor.map(hash_file, file_paths): + for file_hash in tqdm( + executor.map(hash_file, file_paths), + total=len(file_paths), + desc="Hashing files", + ): file_hashes.append(file_hash) return np.array(file_paths), np.array(file_hashes) From 3772b89f77847055047787472cb433b78adbb54b Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 28 Jul 2023 22:31:52 +0200 Subject: [PATCH 02/13] Added GUI package to lockfile. Also added github ref to PYPI page. --- poetry.lock | 172 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + 2 files changed, 173 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 4e53bb0..5652348 100644 --- a/poetry.lock +++ b/poetry.lock @@ -94,6 +94,17 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colored" +version = "1.4.4" +description = "Simple library for color and formatting to terminal" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "colored-1.4.4.tar.gz", hash = "sha256:04ff4d4dd514274fe3b99a21bb52fb96f2688c01e93fba7bef37221e7cb56ce0"}, +] + [[package]] name = "dill" version = "0.3.7" @@ -124,6 +135,25 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "gooey" +version = "1.0.8.1" +description = "Turn (almost) any command line program into a full GUI application with one line" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "Gooey-1.0.8.1-py2.py3-none-any.whl", hash = "sha256:222793cf4a5dd2f9d5c3174ad7369c3df541d91a49e54d51bbee95745ed75ae2"}, + {file = "Gooey-1.0.8.1.tar.gz", hash = "sha256:08d6bf534f4d50d50dafba5cfc68dcf31a6e9eeef13a94cbe3ea17c4e45c4671"}, +] + +[package.dependencies] +colored = ">=1.3.93" +Pillow = ">=4.3.0" +psutil = ">=5.4.2" +pygtrie = ">=2.3.3" +wxpython = ">=4.1.0" + [[package]] name = "iniconfig" version = "2.0.0" @@ -401,6 +431,76 @@ files = [ {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] +[[package]] +name = "pillow" +version = "10.0.0" +description = "Python Imaging Library (Fork)" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Pillow-10.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891"}, + {file = "Pillow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992"}, + {file = "Pillow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d"}, + {file = "Pillow-10.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f"}, + {file = "Pillow-10.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5"}, + {file = "Pillow-10.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199"}, + {file = "Pillow-10.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7"}, + {file = "Pillow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017"}, + {file = "Pillow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3"}, + {file = "Pillow-10.0.0.tar.gz", hash = "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] + [[package]] name = "platformdirs" version = "3.9.1" @@ -433,6 +533,45 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "psutil" +version = "5.9.5" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pygtrie" +version = "2.5.0" +description = "A pure Python trie data structure implementation." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "pygtrie-2.5.0-py3-none-any.whl", hash = "sha256:8795cda8105493d5ae159a5bef313ff13156c5d4d72feddefacaad59f8c8ce16"}, + {file = "pygtrie-2.5.0.tar.gz", hash = "sha256:203514ad826eb403dab1d2e2ddd034e0d1534bbe4dbe0213bb0593f66beba4e2"}, +] + [[package]] name = "pylint" version = "2.17.4" @@ -679,7 +818,38 @@ files = [ {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] +[[package]] +name = "wxpython" +version = "4.2.1" +description = "Cross platform GUI toolkit for Python, \"Phoenix\" version" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "wxPython-4.2.1-cp310-cp310-macosx_10_10_universal2.whl", hash = "sha256:3fd606d10db694c29f712f13dc3d3179d0204a71f6c1fbb5fcee859d03e9ff97"}, + {file = "wxPython-4.2.1-cp310-cp310-win32.whl", hash = "sha256:5b233c39d7bfb53b9c4928ee7c86d626f1f7a716a6dfc3acc152bb437e658751"}, + {file = "wxPython-4.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:bb7988814e706eb00792589d606d317da5661dcffbb09263cd20da956c46bcbb"}, + {file = "wxPython-4.2.1-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:e87960d1963e291d4009c8547c85716b7d5325eb828cd183fdb28d441d6c3787"}, + {file = "wxPython-4.2.1-cp311-cp311-win32.whl", hash = "sha256:a41f3e03c3bfbe80864d7456f5c1236991fa937eedb60b986bd67e1bb47b7c3d"}, + {file = "wxPython-4.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:3062b4d2f5d6dcf1d59983797fc067270a5ce829a918d52a516212e798f0c9ae"}, + {file = "wxPython-4.2.1-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:957a6e7cc68a8e4d7ca49c72a691b6efd5684040f4f03b112d0122e7ab470497"}, + {file = "wxPython-4.2.1-cp312-cp312-win32.whl", hash = "sha256:8d846a785cd33c31e7eb42038eb159c88977d38208496b8322d14aef107f3eec"}, + {file = "wxPython-4.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:1ca327c877f276b33e2c4b6cb8417964305ee505e2509fb2000851d48b82328f"}, + {file = "wxPython-4.2.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:a7eefbc7fe7ac86479d814302711f8f118ba214229aa2a6d789fb888ee79abaa"}, + {file = "wxPython-4.2.1-cp38-cp38-win32.whl", hash = "sha256:ff0423e84a5aaa203e7c1c3d36f8417c54cb4cd8849a43b3c917ac3bfafa4ad7"}, + {file = "wxPython-4.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:903f45131107802b38c0b5d0e964a2ce0295b67e8c3d7659821ac5b26a35658b"}, + {file = "wxPython-4.2.1-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:a15b76dad81f9bd6ceadf00fbede9aff9e09859a9aa698c53e8bd56b95d2effc"}, + {file = "wxPython-4.2.1-cp39-cp39-win32.whl", hash = "sha256:c4db6d7f054d76cb1532dbd5da81b87b7a90dc9fdf18a3540063b2f8f5f3e663"}, + {file = "wxPython-4.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:0595fdd133a552a232a99759b87891136a4500c806bbea68b888e7b6f2c9bbea"}, + {file = "wxPython-4.2.1.tar.gz", hash = "sha256:e48de211a6606bf072ec3fa778771d6b746c00b7f4b970eb58728ddf56d13d5c"}, +] + +[package.dependencies] +numpy = {version = "*", markers = "python_version >= \"3.0\" and python_version < \"3.12\""} +pillow = "*" +six = "*" + [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "1ee00d375e8df8d2231df24b8efb5262683ded9cbd03c011dbae9867b45d43c2" +content-hash = "e9652a0f60646f34b9b92677116a06acdd8fb2982b5499c475fba406b2f6303d" diff --git a/pyproject.toml b/pyproject.toml index 66475f0..aca4b8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.3" description = "Command line utility that looks for specific BIOS files for libretro cores and, if found, refactors them to the expected format (i.e. name and directory structure)." authors = ["Jasper "] license = "GNU General Public License v3.0" +homepage = "https://github.com/jaspersiebring/libretro_finder" readme = "README.md" packages = [ {include = "libretro_finder"}, @@ -18,6 +19,7 @@ python = "^3.8" numpy = "^1.15.0" pandas = ">=1.1.5" tqdm = "^4.65.0" +gooey = "^1.0.8.1" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" From ea470ed95252a4a4dc8308a90669d0c543fe18e1 Mon Sep 17 00:00:00 2001 From: Jasper Date: Sat, 29 Jul 2023 02:51:33 +0200 Subject: [PATCH 03/13] Added GUI around argparser with outputdir defaulting to retroarch's system dir (if found). New VDF dep (reading Steam's libraryfolders.vdf for pathing) --- README.md | 1 - libretro_finder/main.py | 30 ++++++--------- libretro_finder/utils.py | 80 +++++++++++++++++++++++++++++++++++++++- poetry.lock | 14 ++++++- pyproject.toml | 1 + tests/test_main.py | 8 ++-- tests/test_utils.py | 17 ++++++++- 7 files changed, 124 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index fcf459b..5ffdaec 100644 --- a/README.md +++ b/README.md @@ -64,5 +64,4 @@ positional arguments: optional arguments: -h, --help show this help message and exit - -g GLOB, --glob GLOB Glob pattern to use for file matching ```` \ No newline at end of file diff --git a/libretro_finder/main.py b/libretro_finder/main.py index cb7d28f..e094fd1 100644 --- a/libretro_finder/main.py +++ b/libretro_finder/main.py @@ -1,14 +1,14 @@ -import argparse import shutil import pathlib import numpy as np +from gooey import Gooey, GooeyParser from config import SYSTEMS as system_df -from libretro_finder.utils import match_arrays, recursive_hash +from libretro_finder.utils import match_arrays, recursive_hash, RETROARCH_PATH def organize( - search_dir: pathlib.Path, output_dir: pathlib.Path, glob: str = "*" + search_dir: pathlib.Path, output_dir: pathlib.Path ) -> None: """ Non-destructive function that finds, copies and refactors files to the format expected by @@ -17,14 +17,13 @@ def organize( :param search_dir: :param output_dir: - :param glob: :param overwrite: :return: """ # Indexing files to be checked for matching MD5 checksums output_dir.mkdir(parents=True, exist_ok=True) - file_paths, file_hashes = recursive_hash(directory=search_dir, glob=glob) + file_paths, file_hashes = recursive_hash(directory=search_dir) # Element-wise matching of files against libretro's files matching_values, file_indices, system_indices = match_arrays( @@ -70,32 +69,25 @@ def organize( shutil.copy(src=srcs[i], dst=dst) - +@Gooey(program_name="LibretroFinder") def main() -> None: """Simple argparse wrapper for packaging.""" - parser = argparse.ArgumentParser( + + parser = GooeyParser( description="CLI that finds, copies and refactors BIOS files " "to the format expected by libretro (i.e. name and directory structure).", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument("search_dir", help="Directory to look for BIOS files", type=str) - parser.add_argument( - "output_dir", help="Directory to refactor found BIOS files to", type=str ) + parser.add_argument("search_dir", help="Directory to look for BIOS files", type=str, widget='DirChooser') parser.add_argument( - "-g", - "--glob", - help="Glob pattern to use for file matching", - type=str, - default="*", + "output_dir", help="Directory to refactor found BIOS files to", type=str, widget='DirChooser', default = str(RETROARCH_PATH) ) args = vars(parser.parse_args()) search_directory = pathlib.Path(args["search_dir"]) output_directory = pathlib.Path(args["output_dir"]) - search_glob = args["glob"] + - organize(search_dir=search_directory, output_dir=output_directory, glob=search_glob) + organize(search_dir=search_directory, output_dir=output_directory) if __name__ == "__main__": diff --git a/libretro_finder/utils.py b/libretro_finder/utils.py index 7117995..5e457d3 100644 --- a/libretro_finder/utils.py +++ b/libretro_finder/utils.py @@ -1,9 +1,13 @@ +import os import concurrent.futures import hashlib import pathlib -from typing import Tuple +from typing import Tuple, Optional, List from tqdm import tqdm import numpy as np +import vdf +import platform +from string import ascii_uppercase from config import MAX_BIOS_BYTES @@ -71,3 +75,77 @@ def match_arrays( matching_values = np.unique(array_a[indices_a]) return matching_values, indices_a, indices_b + + +def check_env_var(var_name: str) -> Optional[str]: + """ + This function checks if a specific environment variable exists in the system. + + Args: + var_name (str): The name of the environment variable to check. + + Returns: + Optional[str]: The value of the environment variable if it exists, None otherwise. + """ + + if var_name in os.environ: + return os.environ[var_name] + else: + return None + +def list_steam_libraries(path: pathlib.Path) -> List[pathlib.Path]: + library_paths = [] + with open(path, 'r', encoding="utf-8") as src: + library_info = vdf.parse(src) + for key in library_info['libraryfolders'].keys(): + library_path = pathlib.Path(library_info['libraryfolders'][key]['path']) + if library_path.exists(): + library_paths.append(library_path) + + return library_paths + +def find_retroarch() -> Optional[pathlib.Path]: + paths_to_check = [] + system = platform.system() + print("Locating RetroArch..") + + if system == "Windows": + system_glob = "RetroArch*/system" + # Non-Steam + drives = [pathlib.Path(f"{drive}:").resolve() for drive in ascii_uppercase if pathlib.Path(f"{drive}:").exists()] + for drive in drives: + paths_to_check.append(drive) + + env_vars = ["PROGRAMFILES(X86)", "PROGRAMFILES"] + for env_var in env_vars: + env_var = check_env_var(var_name = env_var) + env_path = pathlib.Path(env_var) + + if env_path.exists(): + paths_to_check.append(env_path) + + # adding steam libraries + vdf_path = env_path / pathlib.Path("Steam", "steamapps", "libraryfolders.vdf") + if vdf_path.exists(): + for library_path in list_steam_libraries(path=vdf_path): + paths_to_check.append(library_path / pathlib.Path("steamapps/common")) + + elif system == "Linux": + # system_glob = "retroarch/system" + # home = pathlib.Path.home() + # paths_to_check.append(home / pathlib.Path(".config")) + #Linux: ~/.local/share/Steam/steamapps/libraryfolders.vdf + raise NotImplementedError("To be implemented..") + elif system == "Darwin": + #MacOS: ~/Library/Application Support/Steam/steamapps/libraryfolders.vdf + raise NotImplementedError("To be implemented..") + + # checking for retroarch/system (one level down) + for path_to_check in paths_to_check: + # glob is needed for installers (e.g. RetroArch-Win32) + for path in path_to_check.glob(system_glob): + return path + return None + +# path to retroarch installation (if found) +RETROARCH_PATH = find_retroarch() \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 5652348..343e722 100644 --- a/poetry.lock +++ b/poetry.lock @@ -733,6 +733,18 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] +[[package]] +name = "vdf" +version = "3.4" +description = "Library for working with Valve's VDF text format" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "vdf-3.4-py2.py3-none-any.whl", hash = "sha256:68c1a125cc49e343d535af2dd25074e9cb0908c6607f073947c4a04bbe234534"}, + {file = "vdf-3.4.tar.gz", hash = "sha256:fd5419f41e07a1009e5ffd027c7dcbe43d1f7e8ef453aeaa90d9d04b807de2af"}, +] + [[package]] name = "wrapt" version = "1.15.0" @@ -852,4 +864,4 @@ six = "*" [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "e9652a0f60646f34b9b92677116a06acdd8fb2982b5499c475fba406b2f6303d" +content-hash = "a330dc3081e05ea78c158a4bc1cfb5498743d720c96028942d0aece962372c18" diff --git a/pyproject.toml b/pyproject.toml index aca4b8d..eeb31d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ numpy = "^1.15.0" pandas = ">=1.1.5" tqdm = "^4.65.0" gooey = "^1.0.8.1" +vdf = "^3.4" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" diff --git a/tests/test_main.py b/tests/test_main.py index 38a9a42..74dd1f9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -34,7 +34,7 @@ def test_matching( # swapping out system_df to the one generated from setup_files # this is needed because we can't include actual bios files for testing monkeypatch.setattr("libretro_finder.main.system_df", bios_lut) - organize(search_dir=bios_dir, output_dir=output_dir, glob="*") + organize(search_dir=bios_dir, output_dir=output_dir) # verifying correct output output_paths = list(output_dir.rglob(pattern="*")) @@ -72,7 +72,7 @@ def test_non_matching(self, setup_files, tmpdir_factory: TempdirFactory) -> None assert len(output_paths) == 0 # running organize with libretro's dataframe - organize(search_dir=bios_dir, output_dir=output_dir, glob="*") + organize(search_dir=bios_dir, output_dir=output_dir) # checking if still empty assert len(list(output_dir.rglob("*"))) == 0 @@ -90,7 +90,7 @@ def test_empty(self, tmpdir_factory: TempdirFactory) -> None: assert len(list(output_dir.rglob("*"))) == 0 # checking if still empty - organize(search_dir=input_dir, output_dir=output_dir, glob="*") + organize(search_dir=input_dir, output_dir=output_dir) assert len(list(output_dir.rglob("*"))) == 0 def test_same_input( @@ -111,7 +111,7 @@ def test_same_input( # swapping out system_df to the one generated from setup_files # this is needed because we can't include actual bios files for testing monkeypatch.setattr("libretro_finder.main.system_df", bios_lut) - organize(search_dir=bios_dir, output_dir=bios_dir, glob="*") + organize(search_dir=bios_dir, output_dir=bios_dir) # verifying correct output output_paths = list(bios_dir.rglob(pattern="*")) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5a43be8..56f0a78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,7 +8,7 @@ import pytest from pytest import TempdirFactory, TempPathFactory -from libretro_finder.utils import hash_file, match_arrays, recursive_hash +from libretro_finder.utils import hash_file, match_arrays, recursive_hash, check_env_var from tests import TEST_BYTES, TEST_SAMPLE_SIZE from tests.fixtures import setup_files @@ -134,3 +134,18 @@ def test_nptype(self): assert np.size(matching_values) > 0 assert np.all(np.isin(np.array(["a", "b"]), matching_values)) assert np.array_equal(array_a[indices_a], array_b[indices_b]) + + +class Test_check_env_var: + def test_check_env_var_exists(self): + os.environ['TEST_VAR'] = 'test_value' + assert check_env_var('TEST_VAR') == 'test_value' + + def test_check_env_var_not_exists(self): + if 'NON_EXISTENT_VAR' in os.environ: + del os.environ['NON_EXISTENT_VAR'] + assert check_env_var('NON_EXISTENT_VAR') == None + + def test_check_env_var_none(self): + with pytest.raises(TypeError): + check_env_var(None) \ No newline at end of file From ec597fade3f28f46c7e2d4d5b4af60e6ef286c7c Mon Sep 17 00:00:00 2001 From: Jasper Date: Sat, 29 Jul 2023 17:53:10 +0200 Subject: [PATCH 04/13] GUI functionality and docs. --- config/__init__.py | 10 +++++---- libretro_finder/main.py | 40 +++++++++++++++++++++++----------- libretro_finder/utils.py | 47 +++++++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index bc31ecb..6131112 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -3,9 +3,8 @@ import urllib.request import pandas as pd +from libretro_finder.utils import find_retroarch -# not expecting BIOS files over 15mb -MAX_BIOS_BYTES = 15728640 SEED = 0 # Pulling all BIOS names and hashes from Libretro's system.dat (https://docs.libretro.com/) @@ -19,9 +18,9 @@ print("Done.") # Parsing Libretro's system.dat and formatting as pandas dataframe -index = 0 # pylint: disable=invalid-name +index = 0 # pylint: disable=invalid-name SYSTEMS = [] -with open(FILE_PATH, "r", encoding='utf-8') as file: +with open(FILE_PATH, "r", encoding="utf-8") as file: for line in file: line = line.strip() if line.startswith("comment"): @@ -45,3 +44,6 @@ # join dfs and drop features without checksums SYSTEMS = pd.concat(SYSTEMS) SYSTEMS = SYSTEMS[~SYSTEMS["md5"].isnull()].reset_index(drop=True) + +# path to retroarch/system (if found) +RETROARCH_PATH = find_retroarch() diff --git a/libretro_finder/main.py b/libretro_finder/main.py index e094fd1..66da5f3 100644 --- a/libretro_finder/main.py +++ b/libretro_finder/main.py @@ -4,12 +4,11 @@ from gooey import Gooey, GooeyParser from config import SYSTEMS as system_df -from libretro_finder.utils import match_arrays, recursive_hash, RETROARCH_PATH +from config import RETROARCH_PATH +from libretro_finder.utils import match_arrays, recursive_hash -def organize( - search_dir: pathlib.Path, output_dir: pathlib.Path -) -> None: +def organize(search_dir: pathlib.Path, output_dir: pathlib.Path) -> None: """ Non-destructive function that finds, copies and refactors files to the format expected by libretro (and its cores). This is useful if you source your BIOS files from many different @@ -69,26 +68,41 @@ def organize( shutil.copy(src=srcs[i], dst=dst) -@Gooey(program_name="LibretroFinder") + +@Gooey(program_name="LibretroFinder", default_size=(610, 530), required_cols=1) def main() -> None: """Simple argparse wrapper for packaging.""" - + parser = GooeyParser( - description="CLI that finds, copies and refactors BIOS files " - "to the format expected by libretro (i.e. name and directory structure).", + description="Locate and prepare your BIOS files for libretro.", + ) + parser.add_argument( + "Search directory", + help="Where to look for BIOS files", + type=pathlib.Path, + widget="DirChooser", ) - parser.add_argument("search_dir", help="Directory to look for BIOS files", type=str, widget='DirChooser') parser.add_argument( - "output_dir", help="Directory to refactor found BIOS files to", type=str, widget='DirChooser', default = str(RETROARCH_PATH) + "Output directory", + help="Where to output refactored BIOS files (defaults to ./retroarch/system)", + type=pathlib.Path, + widget="DirChooser", + default=str(RETROARCH_PATH) if RETROARCH_PATH else None, ) args = vars(parser.parse_args()) - search_directory = pathlib.Path(args["search_dir"]) - output_directory = pathlib.Path(args["output_dir"]) - + search_directory = args["Search directory"] + output_directory = args["Output directory"] + + if not search_directory.exists(): + raise FileNotFoundError("Search directory does not exist..") + elif not search_directory.is_dir(): + raise NotADirectoryError("Search directory needs to be a directory..") organize(search_dir=search_directory, output_dir=output_directory) if __name__ == "__main__": main() + + # diff --git a/libretro_finder/utils.py b/libretro_finder/utils.py index 5e457d3..2c51610 100644 --- a/libretro_finder/utils.py +++ b/libretro_finder/utils.py @@ -2,14 +2,15 @@ import concurrent.futures import hashlib import pathlib -from typing import Tuple, Optional, List +from typing import Tuple, Optional, List, Union from tqdm import tqdm import numpy as np import vdf import platform from string import ascii_uppercase -from config import MAX_BIOS_BYTES +# not expecting BIOS files over 15mb +MAX_BIOS_BYTES = 15728640 def hash_file(file_path: pathlib.Path) -> str: @@ -87,57 +88,66 @@ def check_env_var(var_name: str) -> Optional[str]: Returns: Optional[str]: The value of the environment variable if it exists, None otherwise. """ - + if var_name in os.environ: return os.environ[var_name] else: return None - + + def list_steam_libraries(path: pathlib.Path) -> List[pathlib.Path]: library_paths = [] - with open(path, 'r', encoding="utf-8") as src: + with open(path, "r", encoding="utf-8") as src: library_info = vdf.parse(src) - for key in library_info['libraryfolders'].keys(): - library_path = pathlib.Path(library_info['libraryfolders'][key]['path']) + for key in library_info["libraryfolders"].keys(): + library_path = pathlib.Path(library_info["libraryfolders"][key]["path"]) if library_path.exists(): library_paths.append(library_path) return library_paths + def find_retroarch() -> Optional[pathlib.Path]: paths_to_check = [] system = platform.system() - print("Locating RetroArch..") - + if system == "Windows": system_glob = "RetroArch*/system" # Non-Steam - drives = [pathlib.Path(f"{drive}:").resolve() for drive in ascii_uppercase if pathlib.Path(f"{drive}:").exists()] + drives = [ + pathlib.Path(f"{drive}:").resolve() + for drive in ascii_uppercase + if pathlib.Path(f"{drive}:").exists() + ] for drive in drives: paths_to_check.append(drive) - + env_vars = ["PROGRAMFILES(X86)", "PROGRAMFILES"] - for env_var in env_vars: - env_var = check_env_var(var_name = env_var) + for env_var in env_vars: + env_var = check_env_var(var_name=env_var) env_path = pathlib.Path(env_var) if env_path.exists(): paths_to_check.append(env_path) # adding steam libraries - vdf_path = env_path / pathlib.Path("Steam", "steamapps", "libraryfolders.vdf") + vdf_path = env_path / pathlib.Path( + "Steam", "steamapps", "libraryfolders.vdf" + ) if vdf_path.exists(): for library_path in list_steam_libraries(path=vdf_path): - paths_to_check.append(library_path / pathlib.Path("steamapps/common")) + paths_to_check.append( + library_path / pathlib.Path("steamapps/common") + ) elif system == "Linux": # system_glob = "retroarch/system" # home = pathlib.Path.home() # paths_to_check.append(home / pathlib.Path(".config")) - #Linux: ~/.local/share/Steam/steamapps/libraryfolders.vdf + # Linux: ~/.local/share/Steam/steamapps/libraryfolders.vdf raise NotImplementedError("To be implemented..") elif system == "Darwin": - #MacOS: ~/Library/Application Support/Steam/steamapps/libraryfolders.vdf + # MacOS: ~/Library/Application Support/Steam/steamapps/libraryfolders.vdf raise NotImplementedError("To be implemented..") # checking for retroarch/system (one level down) @@ -147,5 +157,6 @@ def find_retroarch() -> Optional[pathlib.Path]: return path return None + # path to retroarch installation (if found) -RETROARCH_PATH = find_retroarch() \ No newline at end of file +RETROARCH_PATH = find_retroarch() From aa2c5cfa226a4193f9353669d3fd43096f1c3271 Mon Sep 17 00:00:00 2001 From: Jasper Date: Sat, 29 Jul 2023 17:53:45 +0200 Subject: [PATCH 05/13] Added pyinstaller to poetry.lock in preparation for executable build --- poetry.lock | 118 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 3 +- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 343e722..8b7df25 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,17 @@ # This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +[[package]] +name = "altgraph" +version = "0.17.3" +description = "Python graph (network) package" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "altgraph-0.17.3-py2.py3-none-any.whl", hash = "sha256:c8ac1ca6772207179ed8003ce7687757c04b0b71536f81e2ac5755c6226458fe"}, + {file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"}, +] + [[package]] name = "astroid" version = "2.15.6" @@ -230,6 +242,21 @@ files = [ {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, ] +[[package]] +name = "macholib" +version = "1.16.2" +description = "Mach-O header analysis and editing" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "macholib-1.16.2-py2.py3-none-any.whl", hash = "sha256:44c40f2cd7d6726af8fa6fe22549178d3a4dfecc35a9cd15ea916d9c83a688e0"}, + {file = "macholib-1.16.2.tar.gz", hash = "sha256:557bbfa1bb255c20e9abafe7ed6cd8046b48d9525db2f9b77d3122a63a2a8bf8"}, +] + +[package.dependencies] +altgraph = ">=0.17" + [[package]] name = "mccabe" version = "0.7.0" @@ -431,6 +458,18 @@ files = [ {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, ] +[[package]] +name = "pefile" +version = "2023.2.7" +description = "Python PE parsing module" +category = "main" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, +] + [[package]] name = "pillow" version = "10.0.0" @@ -572,6 +611,52 @@ files = [ {file = "pygtrie-2.5.0.tar.gz", hash = "sha256:203514ad826eb403dab1d2e2ddd034e0d1534bbe4dbe0213bb0593f66beba4e2"}, ] +[[package]] +name = "pyinstaller" +version = "5.13.0" +description = "PyInstaller bundles a Python application and all its dependencies into a single package." +category = "main" +optional = false +python-versions = "<3.13,>=3.7" +files = [ + {file = "pyinstaller-5.13.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:7fdd319828de679f9c5e381eff998ee9b4164bf4457e7fca56946701cf002c3f"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0df43697c4914285ecd333be968d2cd042ab9b2670124879ee87931d2344eaf5"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_i686.whl", hash = "sha256:28d9742c37e9fb518444b12f8c8ab3cb4ba212d752693c34475c08009aa21ccf"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e5fb17de6c325d3b2b4ceaeb55130ad7100a79096490e4c5b890224406fa42f4"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:78975043edeb628e23a73fb3ef0a273cda50e765f1716f75212ea3e91b09dede"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:cd7d5c06f2847195a23d72ede17c60857d6f495d6f0727dc6c9bc1235f2eb79c"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:24009eba63cfdbcde6d2634e9c87f545eb67249ddf3b514e0cd3b2cdaa595828"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1fde4381155f21d6354dc450dcaa338cd8a40aaacf6bd22b987b0f3e1f96f3ee"}, + {file = "pyinstaller-5.13.0-py3-none-win32.whl", hash = "sha256:2d03419904d1c25c8968b0ad21da0e0f33d8d65716e29481b5bd83f7f342b0c5"}, + {file = "pyinstaller-5.13.0-py3-none-win_amd64.whl", hash = "sha256:9fc27c5a853b14a90d39c252707673c7a0efec921cd817169aff3af0fca8c127"}, + {file = "pyinstaller-5.13.0-py3-none-win_arm64.whl", hash = "sha256:3a331951f9744bc2379ea5d65d36f3c828eaefe2785f15039592cdc08560b262"}, + {file = "pyinstaller-5.13.0.tar.gz", hash = "sha256:5e446df41255e815017d96318e39f65a3eb807e74a796c7e7ff7f13b6366a2e9"}, +] + +[package.dependencies] +altgraph = "*" +macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2021.4" +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} +setuptools = ">=42.0.0" + +[package.extras] +encryption = ["tinyaes (>=1.0.0)"] +hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2023.6" +description = "Community maintained hooks for PyInstaller" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyinstaller-hooks-contrib-2023.6.tar.gz", hash = "sha256:596a72009d8692b043e0acbf5e1b476d93149900142ba01845dded91a0770cb5"}, + {file = "pyinstaller_hooks_contrib-2023.6-py2.py3-none-any.whl", hash = "sha256:aa6d7d038814df6aa7bec7bdbebc7cb4c693d3398df858f6062957f0797d397b"}, +] + [[package]] name = "pylint" version = "2.17.4" @@ -652,6 +737,35 @@ files = [ {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, +] + +[[package]] +name = "setuptools" +version = "68.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, + {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -863,5 +977,5 @@ six = "*" [metadata] lock-version = "2.0" -python-versions = "^3.8" -content-hash = "a330dc3081e05ea78c158a4bc1cfb5498743d720c96028942d0aece962372c18" +python-versions = ">=3.8,<3.13" +content-hash = "1d6b7c17aa98f6b27237d90f270ed1e336ad27f3fb0ffaa9b2d95fcdffd232b4" diff --git a/pyproject.toml b/pyproject.toml index eeb31d5..2f645cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,13 @@ packages = [ libretro_finder = "libretro_finder.main:main" [tool.poetry.dependencies] -python = "^3.8" +python = ">=3.8,<3.13" numpy = "^1.15.0" pandas = ">=1.1.5" tqdm = "^4.65.0" gooey = "^1.0.8.1" vdf = "^3.4" +pyinstaller = "^5.13.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" From 8eb417d94c1059631e47a432c4952d7d25374a9c Mon Sep 17 00:00:00 2001 From: Jasper Date: Sat, 29 Jul 2023 18:40:44 +0200 Subject: [PATCH 06/13] Embedded image --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5ffdaec..02a5d3e 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ Simple command line utility that recursively looks for specific BIOS files for R This repository does **NOT** include the BIOS files themselves. +# GUI (preview) +![image](https://github.com/jaspersiebring/libretro_finder/assets/25051531/d7bd3e1a-80fd-441f-a425-0bd5623d1e72) + + ## Installation Available through the Python Package Index (PyPI): From c636919c25dadb05a886c7bbb890a41d6671bc89 Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 19:22:36 +0200 Subject: [PATCH 07/13] moved pyinstaller to dev deps --- poetry.lock | 16 ++++++++-------- pyproject.toml | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8b7df25..deb0b3d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4,7 +4,7 @@ name = "altgraph" version = "0.17.3" description = "Python graph (network) package" -category = "main" +category = "dev" optional = false python-versions = "*" files = [ @@ -246,7 +246,7 @@ files = [ name = "macholib" version = "1.16.2" description = "Mach-O header analysis and editing" -category = "main" +category = "dev" optional = false python-versions = "*" files = [ @@ -462,7 +462,7 @@ files = [ name = "pefile" version = "2023.2.7" description = "Python PE parsing module" -category = "main" +category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -615,7 +615,7 @@ files = [ name = "pyinstaller" version = "5.13.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." -category = "main" +category = "dev" optional = false python-versions = "<3.13,>=3.7" files = [ @@ -649,7 +649,7 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] name = "pyinstaller-hooks-contrib" version = "2023.6" description = "Community maintained hooks for PyInstaller" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -741,7 +741,7 @@ files = [ name = "pywin32-ctypes" version = "0.2.2" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" -category = "main" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -753,7 +753,7 @@ files = [ name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -978,4 +978,4 @@ six = "*" [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.13" -content-hash = "1d6b7c17aa98f6b27237d90f270ed1e336ad27f3fb0ffaa9b2d95fcdffd232b4" +content-hash = "d60c8bb05bd513b9f754f8e6fb352f1ac0023f796a2c6e0ab26b2e34b81bdd96" diff --git a/pyproject.toml b/pyproject.toml index 2f645cc..fd7e11f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "libretro-finder" version = "0.1.3" description = "Command line utility that looks for specific BIOS files for libretro cores and, if found, refactors them to the expected format (i.e. name and directory structure)." -authors = ["Jasper "] +authors = ["Jasper Siebring "] license = "GNU General Public License v3.0" homepage = "https://github.com/jaspersiebring/libretro_finder" readme = "README.md" @@ -21,13 +21,13 @@ pandas = ">=1.1.5" tqdm = "^4.65.0" gooey = "^1.0.8.1" vdf = "^3.4" -pyinstaller = "^5.13.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" pylint = "^2.17.4" black = "^23.7.0" mypy = "^1.4.1" +pyinstaller = "^5.13.0" [build-system] requires = ["poetry-core"] From 6ee6fc8a6e9ab64e170280592e6792152e6b8f18 Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 19:24:40 +0200 Subject: [PATCH 08/13] Ensured 'cli if user passes args else gui'. Also added documentation wherever possible --- config/__init__.py | 8 ++++++ libretro_finder/main.py | 20 ++++++++------- libretro_finder/utils.py | 55 +++++++++++++++++++--------------------- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index 6131112..5672f2f 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -1,3 +1,4 @@ +import sys import pathlib import re import urllib.request @@ -47,3 +48,10 @@ # path to retroarch/system (if found) RETROARCH_PATH = find_retroarch() + + +# 'cli' if user passes arguments else 'start gui' +# Needs to be present before the @Gooey decorator (https://github.com/chriskiehl/Gooey/issues/449) +if len(sys.argv) >= 2: + if not '--ignore-gooey' in sys.argv: + sys.argv.append('--ignore-gooey') \ No newline at end of file diff --git a/libretro_finder/main.py b/libretro_finder/main.py index 66da5f3..3bcc673 100644 --- a/libretro_finder/main.py +++ b/libretro_finder/main.py @@ -5,6 +5,7 @@ from config import SYSTEMS as system_df from config import RETROARCH_PATH +from typing import List, Optional from libretro_finder.utils import match_arrays, recursive_hash @@ -14,9 +15,8 @@ def organize(search_dir: pathlib.Path, output_dir: pathlib.Path) -> None: libretro (and its cores). This is useful if you source your BIOS files from many different places and have them saved them under different names (often with duplicates). - :param search_dir: - :param output_dir: - :param overwrite: + :param search_dir: starting location of recursive search + :param output_dir: path to output directory (will be created if it doesn't exist) :return: """ @@ -70,8 +70,12 @@ def organize(search_dir: pathlib.Path, output_dir: pathlib.Path) -> None: @Gooey(program_name="LibretroFinder", default_size=(610, 530), required_cols=1) -def main() -> None: - """Simple argparse wrapper for packaging.""" +def main(argv : Optional[List[str]] = None) -> None: + """ + A simple command line utility that finds and prepares your BIOS files for all documented + RetroArch cores. If called without any arguments, a simple graphical user interface with + the same functionality will be started (courtesy of Gooey). + """ parser = GooeyParser( description="Locate and prepare your BIOS files for libretro.", @@ -89,7 +93,7 @@ def main() -> None: widget="DirChooser", default=str(RETROARCH_PATH) if RETROARCH_PATH else None, ) - args = vars(parser.parse_args()) + args = vars(parser.parse_args(argv)) search_directory = args["Search directory"] output_directory = args["Output directory"] @@ -103,6 +107,4 @@ def main() -> None: if __name__ == "__main__": - main() - - # + main() \ No newline at end of file diff --git a/libretro_finder/utils.py b/libretro_finder/utils.py index 2c51610..f97ee5d 100644 --- a/libretro_finder/utils.py +++ b/libretro_finder/utils.py @@ -15,7 +15,8 @@ def hash_file(file_path: pathlib.Path) -> str: """ - + This function calculates the MD5 hash of a file. + :param file_path: :return: """ @@ -29,10 +30,11 @@ def recursive_hash( directory: pathlib.Path, glob: str = "*" ) -> Tuple[np.ndarray, np.ndarray]: """ + Calculate the MD5 hash for all files that match the glob pattern (recursively). - :param directory: - :param glob: - :return: + :param directory: Starting directory for the glob pattern matching + :param glob: The glob pattern to match files. Defaults to "*". + :return: An array with the file_paths to selected files and an array with the corresponding MD5 hashes """ file_paths = list(directory.rglob(pattern=glob)) @@ -61,9 +63,10 @@ def match_arrays( """ Element-wise array comparison, returns unique values and matching indices per input array - :param array_a: - :param array_b: - :return: + :param array_a: The first array to compare. + :param array_b: The second array to compare. + :return: A tuple of three numpy arrays: unique matching values, indices of the matching values in + array_a and indices of the matching values in array_b. """ # expecting 1D arrays @@ -78,26 +81,16 @@ def match_arrays( return matching_values, indices_a, indices_b -def check_env_var(var_name: str) -> Optional[str]: +def list_steam_libraries(vdf_path: pathlib.Path) -> List[pathlib.Path]: """ - This function checks if a specific environment variable exists in the system. + Getting paths to steam libraries from libraryfolders.vdf (Valve Data File) - Args: - var_name (str): The name of the environment variable to check. - - Returns: - Optional[str]: The value of the environment variable if it exists, None otherwise. + :param vdf_path: The path to libraryfolders.vdf + :return: List of paths to Steam Library locations """ - if var_name in os.environ: - return os.environ[var_name] - else: - return None - - -def list_steam_libraries(path: pathlib.Path) -> List[pathlib.Path]: library_paths = [] - with open(path, "r", encoding="utf-8") as src: + with open(vdf_path, "r", encoding="utf-8") as src: library_info = vdf.parse(src) for key in library_info["libraryfolders"].keys(): library_path = pathlib.Path(library_info["libraryfolders"][key]["path"]) @@ -108,6 +101,12 @@ def list_steam_libraries(path: pathlib.Path) -> List[pathlib.Path]: def find_retroarch() -> Optional[pathlib.Path]: + """ + Find the path to the RetroArch installation in the system. + + :return: The path to the RetroArch installation if found, None otherwise. + """ + paths_to_check = [] system = platform.system() @@ -124,7 +123,9 @@ def find_retroarch() -> Optional[pathlib.Path]: env_vars = ["PROGRAMFILES(X86)", "PROGRAMFILES"] for env_var in env_vars: - env_var = check_env_var(var_name=env_var) + env_var = os.environ.get(env_var) + if not env_var: + continue env_path = pathlib.Path(env_var) if env_path.exists(): @@ -135,7 +136,7 @@ def find_retroarch() -> Optional[pathlib.Path]: "Steam", "steamapps", "libraryfolders.vdf" ) if vdf_path.exists(): - for library_path in list_steam_libraries(path=vdf_path): + for library_path in list_steam_libraries(vdf_path=vdf_path): paths_to_check.append( library_path / pathlib.Path("steamapps/common") ) @@ -152,11 +153,7 @@ def find_retroarch() -> Optional[pathlib.Path]: # checking for retroarch/system (one level down) for path_to_check in paths_to_check: - # glob is needed for installers (e.g. RetroArch-Win32) + # glob is needed for inconsistent parent naming (e.g. RetroArch-Win32, RetroArch-Win64, retroarch) for path in path_to_check.glob(system_glob): return path return None - - -# path to retroarch installation (if found) -RETROARCH_PATH = find_retroarch() From 00f4625b37e93603c35ea66d163f2118e10e5f24 Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 19:25:40 +0200 Subject: [PATCH 09/13] Switch to pytest's tmp_path and tmp_path_factory (already return pathlib.Paths). --- tests/__init__.py | 2 +- tests/fixtures.py | 21 +++++++++++++-------- tests/test_main.py | 30 +++++++++++++++--------------- tests/test_utils.py | 29 +++++++---------------------- 4 files changed, 36 insertions(+), 46 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d16fd05..1a59497 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,2 @@ TEST_BYTES = 100 -TEST_SAMPLE_SIZE = 10 \ No newline at end of file +TEST_SAMPLE_SIZE = 10 diff --git a/tests/fixtures.py b/tests/fixtures.py index a0aa5fa..5e4a446 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -7,7 +7,7 @@ import pandas as pd import pytest -from pytest import TempdirFactory +from pytest import TempPathFactory from config import SEED from config import SYSTEMS as system_df @@ -15,11 +15,16 @@ @pytest.fixture(scope="function") -def setup_files(tmpdir_factory: TempdirFactory) -> Tuple[pathlib.Path, pd.DataFrame]: - """_summary_ - - :param tmpdir_factory: _description_ - :return: _description_ +def setup_files(tmp_path_factory: TempPathFactory) -> Tuple[pathlib.Path, pd.DataFrame]: + """ + Pytest fixture that creates a temporary directory, populates it with fake BIOS files and + returns the path to said directory to be used in tests that expect BIOS files. The function + also updates the reference pandas dataFrame with the CRC32, MD5, and SHA1 hashes of the + dummy files (needed since hashes for the dummy files won't match the checksums for the actual + BIOS files) + + :param tmp_path_factory: A pytest fixture for creating temporary directories. + :return: A tuple containing the path to the temporary directory and the updated DataFrame. """ dummy_bios_lut = ( @@ -28,7 +33,7 @@ def setup_files(tmpdir_factory: TempdirFactory) -> Tuple[pathlib.Path, pd.DataFr .reset_index(drop=True) ) - temp_dir = tmpdir_factory.mktemp("bios") + temp_dir = tmp_path_factory.mktemp("bios") temp_output_dir = pathlib.Path(temp_dir) for index, row in dummy_bios_lut.iterrows(): @@ -50,4 +55,4 @@ def setup_files(tmpdir_factory: TempdirFactory) -> Tuple[pathlib.Path, pd.DataFr with open(temp_path, "wb") as src: src.write(dummy_bytes) - return temp_output_dir, dummy_bios_lut \ No newline at end of file + return temp_output_dir, dummy_bios_lut diff --git a/tests/test_main.py b/tests/test_main.py index 74dd1f9..c69cadb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,11 +1,11 @@ import os import pathlib import stat - +import pytest import numpy as np -from pytest import MonkeyPatch, TempdirFactory +from pytest import MonkeyPatch, TempdirFactory, TempPathFactory -from libretro_finder.main import organize +from libretro_finder.main import organize, main from libretro_finder.utils import hash_file from tests import TEST_SAMPLE_SIZE from tests.fixtures import setup_files @@ -13,7 +13,7 @@ class Test_organize: def test_matching( - self, setup_files, tmpdir_factory: TempdirFactory, monkeypatch: MonkeyPatch + self, setup_files, tmp_path: pathlib.Path, monkeypatch: MonkeyPatch ) -> None: bios_dir, bios_lut = setup_files assert bios_dir.exists() @@ -24,8 +24,8 @@ def test_matching( assert len(file_paths) == TEST_SAMPLE_SIZE # making output_dir - temp_dir = tmpdir_factory.mktemp("test_matching") - output_dir = pathlib.Path(temp_dir) + output_dir = tmp_path / "test_matching" + output_dir.mkdir() # checking if currently empty output_paths = list(output_dir.rglob(pattern="*")) @@ -52,7 +52,7 @@ def test_matching( assert np.all(np.isin(output_hashes, bios_lut["md5"].values)) assert np.all(np.isin(bios_lut["name"].values, output_names)) - def test_non_matching(self, setup_files, tmpdir_factory: TempdirFactory) -> None: + def test_non_matching(self, setup_files, tmp_path: pathlib.Path) -> None: # pretty much matching but we don't monkeypatch so we're comparing different hashes # same as 'matching' test but without monkeypatching (i.e. different hashes so no matches) bios_dir, _ = setup_files @@ -64,8 +64,8 @@ def test_non_matching(self, setup_files, tmpdir_factory: TempdirFactory) -> None assert len(file_paths) == TEST_SAMPLE_SIZE # making output_dir - temp_dir = tmpdir_factory.mktemp("test_non_matching") - output_dir = pathlib.Path(temp_dir) + output_dir = tmp_path / "test_non_matching" + output_dir.mkdir() # checking if currently empty output_paths = list(output_dir.rglob(pattern="*")) @@ -77,11 +77,11 @@ def test_non_matching(self, setup_files, tmpdir_factory: TempdirFactory) -> None # checking if still empty assert len(list(output_dir.rglob("*"))) == 0 - def test_empty(self, tmpdir_factory: TempdirFactory) -> None: - temp_input_dir = tmpdir_factory.mktemp("input") - temp_output_dir = tmpdir_factory.mktemp("output") - input_dir = pathlib.Path(temp_input_dir) - output_dir = pathlib.Path(temp_output_dir) + def test_empty(self, tmp_path: pathlib.Path) -> None: + input_dir = tmp_path / "input" + output_dir = tmp_path / "output" + input_dir.mkdir() + output_dir.mkdir() # checking if exists but empty assert input_dir.exists() @@ -94,7 +94,7 @@ def test_empty(self, tmpdir_factory: TempdirFactory) -> None: assert len(list(output_dir.rglob("*"))) == 0 def test_same_input( - self, setup_files, tmpdir_factory: TempdirFactory, monkeypatch: MonkeyPatch + self, setup_files, monkeypatch: MonkeyPatch ) -> None: # organize but with (prepopulated) bios_dir as input and output # verifies if non-additive diff --git a/tests/test_utils.py b/tests/test_utils.py index 56f0a78..bc47c44 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,7 +8,7 @@ import pytest from pytest import TempdirFactory, TempPathFactory -from libretro_finder.utils import hash_file, match_arrays, recursive_hash, check_env_var +from libretro_finder.utils import hash_file, match_arrays, recursive_hash from tests import TEST_BYTES, TEST_SAMPLE_SIZE from tests.fixtures import setup_files @@ -18,15 +18,15 @@ def test_libretro_meta_import(): class Test_hash_file: - def test_existing(self, tmp_path: TempPathFactory) -> None: - file_path = tmp_path / "some_file" # type: ignore + def test_existing(self, tmp_path: pathlib.Path) -> None: + file_path = tmp_path / "some_file" random_bytes = os.urandom(TEST_BYTES) with open(file_path, "wb") as src: src.write(random_bytes) _ = hash_file(file_path) - def test_nonexistent(self, tmp_path) -> None: + def test_nonexistent(self, tmp_path: pathlib.Path) -> None: file_path = tmp_path / "some_file" with pytest.raises(FileNotFoundError): hash_file(file_path) @@ -52,9 +52,9 @@ def test_with_fixture(self, setup_files: Tuple[pathlib.Path, pd.DataFrame]) -> N assert np.array_equal(file_names[index_b], bios_lut["name"].values[index_a]) assert np.array_equal(file_hashes[index_b], bios_lut["md5"].values[index_a]) - def test_without_fixture(self, tmpdir_factory: TempdirFactory) -> None: - temp_dir = tmpdir_factory.mktemp("rhash") - temp_output_dir = pathlib.Path(temp_dir) + def test_without_fixture(self, tmp_path: pathlib.Path) -> None: + temp_output_dir = tmp_path / "rhash" + temp_output_dir.mkdir() file_names = [f"system_{i}.bin" for i in range(1, 11)] file_bytes = [os.urandom(TEST_BYTES * i) for i in range(1, 11)] @@ -134,18 +134,3 @@ def test_nptype(self): assert np.size(matching_values) > 0 assert np.all(np.isin(np.array(["a", "b"]), matching_values)) assert np.array_equal(array_a[indices_a], array_b[indices_b]) - - -class Test_check_env_var: - def test_check_env_var_exists(self): - os.environ['TEST_VAR'] = 'test_value' - assert check_env_var('TEST_VAR') == 'test_value' - - def test_check_env_var_not_exists(self): - if 'NON_EXISTENT_VAR' in os.environ: - del os.environ['NON_EXISTENT_VAR'] - assert check_env_var('NON_EXISTENT_VAR') == None - - def test_check_env_var_none(self): - with pytest.raises(TypeError): - check_env_var(None) \ No newline at end of file From 32f749607070bdc4badf337a25503c091b597eae Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 19:30:22 +0200 Subject: [PATCH 10/13] Prepping 0.2.1 release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd7e11f..e839a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "libretro-finder" -version = "0.1.3" +version = "0.2.1" description = "Command line utility that looks for specific BIOS files for libretro cores and, if found, refactors them to the expected format (i.e. name and directory structure)." authors = ["Jasper Siebring "] license = "GNU General Public License v3.0" From 6ca95d3d4544c66df3257913c70a14babe22a1d1 Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 20:57:29 +0200 Subject: [PATCH 11/13] Added tests for cli call with pytest-mock (added as dep). Also added retroarch path searching for linux/macos (incomplete though) --- libretro_finder/utils.py | 33 +++++++++++++++++++++++---------- poetry.lock | 20 +++++++++++++++++++- pyproject.toml | 1 + tests/test_main.py | 37 ++++++++++++++++++++++++++++++++++++- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/libretro_finder/utils.py b/libretro_finder/utils.py index f97ee5d..6bfee9a 100644 --- a/libretro_finder/utils.py +++ b/libretro_finder/utils.py @@ -16,7 +16,7 @@ def hash_file(file_path: pathlib.Path) -> str: """ This function calculates the MD5 hash of a file. - + :param file_path: :return: """ @@ -65,7 +65,7 @@ def match_arrays( :param array_a: The first array to compare. :param array_b: The second array to compare. - :return: A tuple of three numpy arrays: unique matching values, indices of the matching values in + :return: A tuple of three numpy arrays: unique matching values, indices of the matching values in array_a and indices of the matching values in array_b. """ @@ -106,7 +106,7 @@ def find_retroarch() -> Optional[pathlib.Path]: :return: The path to the RetroArch installation if found, None otherwise. """ - + paths_to_check = [] system = platform.system() @@ -142,14 +142,27 @@ def find_retroarch() -> Optional[pathlib.Path]: ) elif system == "Linux": - # system_glob = "retroarch/system" - # home = pathlib.Path.home() - # paths_to_check.append(home / pathlib.Path(".config")) - # Linux: ~/.local/share/Steam/steamapps/libraryfolders.vdf - raise NotImplementedError("To be implemented..") + system_glob = "retroarch/system" + home = pathlib.Path.home() + paths_to_check.append(home / pathlib.Path(".config")) + vdf_path = home / pathlib.Path( + ".local/share/Steam/steamapps/libraryfolders.vdf" + ) + if vdf_path.exists(): + for library_path in list_steam_libraries(vdf_path=vdf_path): + paths_to_check.append(library_path / pathlib.Path("steamapps/common")) + + # requires more testing on actual metal elif system == "Darwin": - # MacOS: ~/Library/Application Support/Steam/steamapps/libraryfolders.vdf - raise NotImplementedError("To be implemented..") + system_glob = "RetroArch.app/Contents/Resources/system" + home = pathlib.Path.home() + paths_to_check.append(home / pathlib.Path("Library/Application Support")) + vdf_path = home / pathlib.Path( + "Library/Application Support/Steam/steamapps/libraryfolders.vdf" + ) + if vdf_path.exists(): + for library_path in list_steam_libraries(vdf_path=vdf_path): + paths_to_check.append(library_path / pathlib.Path("steamapps/common")) # checking for retroarch/system (one level down) for path_to_check in paths_to_check: diff --git a/poetry.lock b/poetry.lock index deb0b3d..425cdce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -710,6 +710,24 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-mock" +version = "3.11.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, + {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -978,4 +996,4 @@ six = "*" [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.13" -content-hash = "d60c8bb05bd513b9f754f8e6fb352f1ac0023f796a2c6e0ab26b2e34b81bdd96" +content-hash = "d32e3cc909ccd2fc2758726be5fba2cc9988663ff7b9fe7287c2ae1fd90c9eb8" diff --git a/pyproject.toml b/pyproject.toml index e839a03..934165e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ pylint = "^2.17.4" black = "^23.7.0" mypy = "^1.4.1" pyinstaller = "^5.13.0" +pytest-mock = "^3.11.1" [build-system] requires = ["poetry-core"] diff --git a/tests/test_main.py b/tests/test_main.py index c69cadb..556b486 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -4,6 +4,8 @@ import pytest import numpy as np from pytest import MonkeyPatch, TempdirFactory, TempPathFactory +from pytest_mock import MockerFixture +from pytest_mock import mocker from libretro_finder.main import organize, main from libretro_finder.utils import hash_file @@ -126,4 +128,37 @@ def test_same_input( assert len(output_paths) == current_len assert bios_lut.shape[0] == len(output_paths) assert np.all(np.isin(output_hashes, bios_lut["md5"].values)) - assert np.all(np.isin(bios_lut["name"].values, output_names)) \ No newline at end of file + assert np.all(np.isin(bios_lut["name"].values, output_names)) + + +class Test_main: + def test_main(self, tmp_path: pathlib.Path, mocker: MockerFixture): + # Mocking the organize function to prevent actual file operations + mock_organize = mocker.patch('libretro_finder.main.organize') + + search_dir = tmp_path / "search" + output_dir = tmp_path / "output" + search_dir.mkdir() + output_dir.mkdir() + + argv = [str(search_dir), str(output_dir)] + main(argv) + mock_organize.assert_called_once_with(search_dir=search_dir, output_dir=output_dir) + + def test_main_search_directory_not_exists(self, tmp_path: pathlib.Path): + output_dir = tmp_path / "output" + output_dir.mkdir() + + argv = ['/path/to/nonexistent/search', str(output_dir)] + with pytest.raises(FileNotFoundError): + main(argv) + + def test_main_search_directory_not_directory(self, tmp_path: pathlib.Path): + file_path = tmp_path / "search.txt" + output_dir = tmp_path / "output" + file_path.touch() + output_dir.mkdir() + + argv = [str(file_path), str(output_dir)] + with pytest.raises(NotADirectoryError): + main(argv) \ No newline at end of file From 00c63739ee37fb650e5af61e21011a3b05eb706a Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 23:49:20 +0200 Subject: [PATCH 12/13] Updated README --- README.md | 82 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 02a5d3e..bd776a6 100644 --- a/README.md +++ b/README.md @@ -3,69 +3,91 @@ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/libretro-finder) ![PyPI - License](https://img.shields.io/pypi/l/libretro-finder) -Simple command line utility that recursively looks for specific BIOS files for RetroArch cores and, if found, refactors them to the expected format as documented by Libretro [here](https://github.com/libretro/libretro-database/blob/4a98ea9726b3954a4e5a940d255bd14c307ddfba/dat/System.dat) (i.e. name and directory structure). This is useful if you source your BIOS files from many different places and have them saved them under different names (often with duplicates). This script is able to find these exact BIOS files by comparing their hashes against their known counterparts. Uses concurrency and vectorization for added performance. +Simple tool that finds and prepares your BIOS files for usage with Libretro (or its RetroArch frontend). + +No more need to manually select, rename and move your BIOS files to some RetroArch installation somewhere, just dump them in LibretroFinder and let it sort it out for you. It does this by generating checksums for your local files (i.e. unique identifiers) and comparing them against their known counterparts as documented by Libretro [here](https://github.com/libretro/libretro-database/blob/4a98ea9726b3954a4e5a940d255bd14c307ddfba/dat/System.dat). It then refactors *copies* of all matching files to the format expected by Libretro (name *and* folder structure). This repository does **NOT** include the BIOS files themselves. -# GUI (preview) -![image](https://github.com/jaspersiebring/libretro_finder/assets/25051531/d7bd3e1a-80fd-441f-a425-0bd5623d1e72) +## Features +- Simple graphical user interface (GUI) +- Scriptable command line interface (CLI) +- Works on Windows, Linux and MacOS +- Available through the Python Package Index (Python >=3.8) +- Available as portable executable (currently only on X86 Windows machines) + +

+ + +

-## Installation -Available through the Python Package Index (PyPI): +# Installation +Installing from the Python Package Index (PyPI): ```` -# with pip +# Install from PYPI with Python's package installer (pip) pip install libretro-finder -# with poetry -poetry add libretro-finder +# [Optional] Install as isolated application through pipx (https://pypa.github.io/pipx/) +pipx install libretro-finder ```` +You can also download the prebuilt standalone executable which includes Python and all the program's dependencies, no installation required (currently only available for X86 Windows machines). See [releases](https://github.com/jaspersiebring/libretro_finder/releases). -## Example of usage: -```` -some_user@some_machine:~ libretro_finder ~/Downloads/bios_files/ ~/Downloads/libretro_bios +

+ +

-89 matching BIOS files were found for 3 unique systems: - Sega - Mega Drive - Genesis (1) - Sony - PlayStation (19) - Sony - PlayStation 2 (69) -100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 89/89 [00:00<00:00, 617.57it/s] -```` +## Example of usage + +#### Command line interface + +If installed with pip, LibretroFinder can be called directly from your preferred terminal by running `libretro_finder`. You can use the tool entirely from your terminal by providing values for the search directory (e.g. `~/Downloads/bios_files/`) and the output directory (`~/.config/retroarch/system/`): -You can also safely use the `system` directory of retroarch as `output_dir` since we never overwrite files (i.e. libretro_finder only adds to your list of BIOS files) ```` some_user@some_machine:~ libretro_finder ~/Downloads/bios_files/ ~/.config/retroarch/system/ - +Hashing files: 100%|█████████████████████████████████████████████████████████████████████████████████| 983/983 [00:00<00:00, 3333.95it/s] 89 matching BIOS files were found for 3 unique systems: Sega - Mega Drive - Genesis (1) Sony - PlayStation (19) Sony - PlayStation 2 (69) -100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 89/89 [00:00<00:00, 617.57it/s] ```` -You can also use libretro's `system` directory as `search_dir` to find all aliased BIOS files (i.e. identical BIOS files that are usable by different cores for the same system but just under different names). +Although the output directory defaults to retroarch's `system` folder (if `retroarch` was found), you can manually specify whatever output folder you want and `libretro_finder` will create it for you. If your path contains spaces, wrap it in double quotes like so: ```` -some_user@some_machine:~ libretro_finder ~/.config/retroarch/system/ ~/.config/retroarch/system/ - +some_user@some_machine:~ libretro_finder "D:\Games\My Roms" "C:\Program Files (x86)\Steam\steamapps\common\RetroArch\system" +Hashing files: 100%|█████████████████████████████████████████████████████████████████████████████████| 983/983 [00:00<00:00, 3333.95it/s] 89 matching BIOS files were found for 3 unique systems: Sega - Mega Drive - Genesis (1) Sony - PlayStation (19) Sony - PlayStation 2 (69) -100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 89/89 [00:00<00:00, 617.57it/s] ```` - -Help page: +No matter what you select as search- or output directory, rest assured that no existing files on your file system will be modified. You can also call `libretro_finder` with `--help` to get some more information on the expected input: ```` some_user@some_machine:~ libretro_finder --help -Local BIOS file scraper for Retroarch +Locate and prepare your BIOS files for libretro. positional arguments: - search_dir Directory to look for BIOS files - output_dir Directory to refactor found BIOS files to + Search directory Where to look for BIOS files + Output directory Where to output refactored BIOS files (defaults to ./retroarch/system) optional arguments: -h, --help show this help message and exit -```` \ No newline at end of file +```` + + +#### Graphical user interface + +If `libretro_finder` is called without any additional arguments, LibretroFinder will start with a graphical interface. This is functionally identical to the CLI version with the only real difference that it automatically tries to set the output directory to retroarch's `system` folder. + +

+ + +

+ + +### Missing features? Have some feedback? Let me know! +- [My Reddit account](https://www.reddit.com/user/qtieb/) +- [My Github account](https://github.com/jaspersiebring) \ No newline at end of file From 3a4269034f3c82e40e6ad10037dac7a2c11e0dc0 Mon Sep 17 00:00:00 2001 From: Jasper Date: Mon, 31 Jul 2023 23:54:11 +0200 Subject: [PATCH 13/13] black formatting --- config/__init__.py | 4 ++-- libretro_finder/main.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index 5672f2f..cef5678 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -53,5 +53,5 @@ # 'cli' if user passes arguments else 'start gui' # Needs to be present before the @Gooey decorator (https://github.com/chriskiehl/Gooey/issues/449) if len(sys.argv) >= 2: - if not '--ignore-gooey' in sys.argv: - sys.argv.append('--ignore-gooey') \ No newline at end of file + if not "--ignore-gooey" in sys.argv: + sys.argv.append("--ignore-gooey") diff --git a/libretro_finder/main.py b/libretro_finder/main.py index 3bcc673..0e8bed8 100644 --- a/libretro_finder/main.py +++ b/libretro_finder/main.py @@ -70,10 +70,10 @@ def organize(search_dir: pathlib.Path, output_dir: pathlib.Path) -> None: @Gooey(program_name="LibretroFinder", default_size=(610, 530), required_cols=1) -def main(argv : Optional[List[str]] = None) -> None: +def main(argv: Optional[List[str]] = None) -> None: """ - A simple command line utility that finds and prepares your BIOS files for all documented - RetroArch cores. If called without any arguments, a simple graphical user interface with + A simple command line utility that finds and prepares your BIOS files for all documented + RetroArch cores. If called without any arguments, a simple graphical user interface with the same functionality will be started (courtesy of Gooey). """ @@ -107,4 +107,4 @@ def main(argv : Optional[List[str]] = None) -> None: if __name__ == "__main__": - main() \ No newline at end of file + main()