From 7ff11322964c22f608162cc8729c236ed258c1a9 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Fri, 29 Sep 2023 13:22:43 +0200 Subject: [PATCH 01/30] added albumentations transform test --- tests/darwin/torch/transform_test.py | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/darwin/torch/transform_test.py diff --git a/tests/darwin/torch/transform_test.py b/tests/darwin/torch/transform_test.py new file mode 100644 index 000000000..8bc974b08 --- /dev/null +++ b/tests/darwin/torch/transform_test.py @@ -0,0 +1,84 @@ +import numpy as np +import pytest +import torch +from albumentations import BboxParams, Compose, HorizontalFlip, Resize +from PIL import Image + +from darwin.torch.transforms import AlbumentationsTransform + +# Sample data +SAMPLE_IMAGE = Image.new("RGB", (100, 100)) +SAMPLE_ANNOTATION = {"boxes": torch.tensor([[25, 25, 75, 75]]), "labels": torch.tensor([1]), "area": torch.tensor([2500.0]), "iscrowd": torch.tensor([0])} + +SAMPLE_ANNOTATION_OOB = { + "boxes": torch.tensor([[25, 25, 105, 105]]), # Out of bounds + "labels": torch.tensor([1]), + "area": torch.tensor([2500.0]), + "iscrowd": torch.tensor([0]), +} + +SAMPLE_ANNOTATION_WITH_MASKS = {**SAMPLE_ANNOTATION, "masks": torch.ones((1, 100, 100))} + +EXAMPLE_TRANSFORM = Compose([HorizontalFlip(p=1)], bbox_params=BboxParams(format="coco", label_fields=["labels"])) +EXAMPLE_TRANSFORM_RESIZE = Compose([Resize(50, 50)], bbox_params=BboxParams(format="coco", label_fields=["labels"])) + + +class TestAlbumentationsTransform: + def test_init(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + assert isinstance(at, AlbumentationsTransform) + + def test_from_path_invalid(self): + with pytest.raises(ValueError): + AlbumentationsTransform.from_path("invalid/path/to/config.yml") + + def test_from_dict_invalid(self): + with pytest.raises(ValueError): + AlbumentationsTransform.from_dict({"invalid": "config"}) + + def test_transformations(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + image, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) + assert annotation["boxes"][0, 0] != SAMPLE_ANNOTATION["boxes"][0, 0] + + def test_transformations_resize(self): + transformations = EXAMPLE_TRANSFORM_RESIZE + at = AlbumentationsTransform(transformations) + image, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) + assert image.shape[:2] == (50, 50) # We only check the height and width + + def test_boxes_out_of_bounds(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + with pytest.raises(ValueError): + _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_OOB) # Expecting the ValueError due to out of bounds + + def test_transform_with_masks(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_WITH_MASKS) + assert "masks" in annotation + assert annotation["masks"].shape[0] == 1 + + def test_area_calculation_with_masks(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_WITH_MASKS) + assert annotation["area"] == torch.sum(annotation["masks"]) + + def test_area_calculation_without_masks(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) + area = annotation["boxes"][0, 2] * annotation["boxes"][0, 3] + + assert torch.isclose(annotation["area"], area.unsqueeze(0), atol=1e-5) # Using isclose for floating point comparison + + def test_iscrowd_unchanged(self): + transformations = EXAMPLE_TRANSFORM + at = AlbumentationsTransform(transformations) + _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) + assert "iscrowd" in annotation + assert annotation["iscrowd"] == SAMPLE_ANNOTATION["iscrowd"] From 88a2750786a0bfa3a85c4328727a894975ea3465 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Fri, 29 Sep 2023 14:27:06 +0200 Subject: [PATCH 02/30] updated poetry file --- poetry.lock | 88 +++++------------------------------------------------ 1 file changed, 8 insertions(+), 80 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7e534aaa3..6d8b9a3f0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "argcomplete" version = "2.1.2" description = "Bash tab completion for argparse" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -23,7 +22,6 @@ test = ["coverage", "flake8", "mypy", "pexpect", "wheel"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -45,7 +43,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -82,7 +79,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -94,7 +90,6 @@ files = [ name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -179,7 +174,6 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -195,7 +189,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -207,7 +200,6 @@ files = [ name = "connected-components-3d" version = "3.12.2" description = "Connected components on 2D and 3D images. Supports multiple labels." -category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -255,7 +247,6 @@ numpy = "*" name = "debugpy" version = "1.6.7.post1" description = "An implementation of the Debug Adapter Protocol for Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -283,7 +274,6 @@ files = [ name = "deprecation" version = "2.1.0" description = "A library to handle automated deprecations" -category = "main" optional = false python-versions = "*" files = [ @@ -298,7 +288,6 @@ packaging = "*" name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -313,7 +302,6 @@ test = ["pytest (>=6)"] name = "flake8" version = "6.1.0" description = "the modular source code checker: pep8 pyflakes and co" -category = "main" optional = true python-versions = ">=3.8.1" files = [ @@ -330,7 +318,6 @@ pyflakes = ">=3.1.0,<3.2.0" name = "flake8-pyproject" version = "1.2.3" description = "Flake8 plug-in loading the configuration from pyproject.toml" -category = "main" optional = true python-versions = ">= 3.6" files = [ @@ -348,7 +335,6 @@ dev = ["pyTest", "pyTest-cov"] name = "humanize" version = "4.6.0" description = "Python humanize utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -366,7 +352,6 @@ tests = ["freezegun", "pytest", "pytest-cov"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -378,7 +363,6 @@ files = [ name = "importlib-metadata" version = "5.2.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -399,7 +383,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -418,7 +401,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -430,7 +412,6 @@ files = [ name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -448,7 +429,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "joblib" version = "1.3.2" description = "Lightweight pipelining with Python functions" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -460,7 +440,6 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -484,7 +463,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -510,7 +488,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -522,7 +499,6 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -534,7 +510,6 @@ files = [ name = "mpire" version = "2.8.0" description = "A Python package for easy multiprocessing, but faster than multiprocessing" -category = "main" optional = false python-versions = "*" files = [ @@ -557,7 +532,6 @@ testing = ["dataclasses", "multiprocess", "multiprocess (>=0.70.15)", "numpy", " name = "mypy" version = "0.991" description = "Optional static typing for Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -609,7 +583,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -621,7 +594,6 @@ files = [ name = "nibabel" version = "5.1.0" description = "Access a multitude of neuroimaging data formats" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -652,7 +624,6 @@ zstd = ["pyzstd (>=0.14.3)"] name = "numpy" version = "1.21.6" description = "NumPy is the fundamental package for array computing with Python." -category = "main" optional = false python-versions = ">=3.7,<3.11" files = [ @@ -693,7 +664,6 @@ files = [ name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" -category = "main" optional = true python-versions = ">=3" files = [ @@ -709,7 +679,6 @@ wheel = "*" name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" -category = "main" optional = true python-versions = ">=3" files = [ @@ -726,7 +695,6 @@ wheel = "*" name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" -category = "main" optional = true python-versions = ">=3" files = [ @@ -742,7 +710,6 @@ wheel = "*" name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" -category = "main" optional = true python-versions = ">=3" files = [ @@ -758,7 +725,6 @@ wheel = "*" name = "opencv-python-headless" version = "4.8.0.76" description = "Wrapper package for OpenCV python bindings." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -773,19 +739,18 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.21.0", markers = "python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, + {version = ">=1.21.0", markers = "python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\" and python_version >= \"3.7\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""}, - {version = ">=1.17.0", markers = "python_version >= \"3.7\""}, - {version = ">=1.17.3", markers = "python_version >= \"3.8\""}, + {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\" and python_version >= \"3.7\" or python_version < \"3.10\" and platform_system != \"Darwin\" and python_version >= \"3.9\" or python_version < \"3.10\" and python_version >= \"3.9\" and platform_machine != \"arm64\" or python_version > \"3.9\" and python_version < \"3.10\""}, + {version = ">=1.17.3", markers = "(platform_system != \"Darwin\" and platform_system != \"Linux\") and python_version >= \"3.8\" and python_version < \"3.9\" or platform_system != \"Darwin\" and python_version >= \"3.8\" and python_version < \"3.9\" and platform_machine != \"aarch64\" or platform_machine != \"arm64\" and python_version >= \"3.8\" and python_version < \"3.9\" and platform_system != \"Linux\" or (platform_machine != \"arm64\" and platform_machine != \"aarch64\") and python_version >= \"3.8\" and python_version < \"3.9\""}, + {version = ">=1.17.0", markers = "(platform_system != \"Darwin\" and platform_system != \"Linux\") and python_version >= \"3.7\" and python_version < \"3.8\" or platform_system != \"Darwin\" and python_version >= \"3.7\" and python_version < \"3.8\" and platform_machine != \"aarch64\" or platform_machine != \"arm64\" and python_version >= \"3.7\" and python_version < \"3.8\" and platform_system != \"Linux\" or (platform_machine != \"arm64\" and platform_machine != \"aarch64\") and python_version >= \"3.7\" and python_version < \"3.8\""}, ] [[package]] name = "orjson" version = "3.9.5" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -855,7 +820,6 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -867,7 +831,6 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -879,7 +842,6 @@ files = [ name = "pillow" version = "9.5.0" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -959,7 +921,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -971,7 +932,6 @@ files = [ name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -990,7 +950,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1009,7 +968,6 @@ testing = ["pytest", "pytest-benchmark"] name = "pycodestyle" version = "2.11.0" description = "Python style guide checker" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1021,7 +979,6 @@ files = [ name = "pydantic" version = "1.10.12" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1074,7 +1031,6 @@ email = ["email-validator (>=1.0.3)"] name = "pyflakes" version = "3.1.0" description = "passive checker of Python programs" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1086,7 +1042,6 @@ files = [ name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1101,7 +1056,6 @@ plugins = ["importlib-metadata"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1138,7 +1092,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1162,7 +1115,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1177,7 +1129,6 @@ cli = ["click (>=5.0)"] name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "main" optional = false python-versions = "*" files = [ @@ -1201,7 +1152,6 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1261,7 +1211,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1283,7 +1232,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1305,7 +1253,6 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rich" version = "13.5.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -1325,7 +1272,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "scikit-learn" version = "1.3.0" description = "A set of python modules for machine learning and data mining" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1368,7 +1314,6 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( name = "scipy" version = "1.10.1" description = "Fundamental algorithms for scientific computing in Python" -category = "main" optional = true python-versions = "<3.12,>=3.8" files = [ @@ -1407,7 +1352,6 @@ test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeo name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1424,7 +1368,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "threadpoolctl" version = "3.2.0" description = "threadpoolctl" -category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1436,7 +1379,6 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1448,7 +1390,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1460,7 +1401,6 @@ files = [ name = "torch" version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -1501,7 +1441,6 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "torchvision" version = "0.14.1" description = "image and video datasets and models for torch deep learning" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1528,7 +1467,7 @@ files = [ [package.dependencies] numpy = "*" -pillow = ">=5.3.0,<8.3.0 || >=8.4.0" +pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" requests = "*" torch = "1.13.1" typing-extensions = "*" @@ -1540,7 +1479,6 @@ scipy = ["scipy"] name = "tqdm" version = "4.66.1" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1561,7 +1499,6 @@ telegram = ["requests"] name = "typed-ast" version = "1.5.5" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1612,7 +1549,6 @@ files = [ name = "types-pyyaml" version = "6.0.12.11" description = "Typing stubs for PyYAML" -category = "main" optional = false python-versions = "*" files = [ @@ -1624,7 +1560,6 @@ files = [ name = "types-requests" version = "2.31.0.2" description = "Typing stubs for requests" -category = "main" optional = false python-versions = "*" files = [ @@ -1639,7 +1574,6 @@ types-urllib3 = "*" name = "types-toml" version = "0.10.8.7" description = "Typing stubs for toml" -category = "main" optional = true python-versions = "*" files = [ @@ -1651,7 +1585,6 @@ files = [ name = "types-urllib3" version = "1.26.25.14" description = "Typing stubs for urllib3" -category = "main" optional = false python-versions = "*" files = [ @@ -1663,7 +1596,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1675,7 +1607,6 @@ files = [ name = "upolygon" version = "0.1.10" description = "Collection of fast polygon operations for DL" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1697,7 +1628,6 @@ numpy = "*" name = "urllib3" version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1715,7 +1645,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "wheel" version = "0.41.1" description = "A built-package format for Python" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1730,7 +1659,6 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1752,4 +1680,4 @@ test = ["flake8-pyproject", "pytest", "responses"] [metadata] lock-version = "2.0" python-versions = ">=3.7.0,<3.11" -content-hash = "bf2caf6db46010fe08b8aac8c522e07985b2a0758c89e830a0f92bdc455c86e3" +content-hash = "791455d63b5366c2242672436020cfb6ce923be62326751b60ac91deea3587a9" From 520f9f38ef2e081090f62ca3e3a2331e5fda0800 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Fri, 29 Sep 2023 14:54:02 +0200 Subject: [PATCH 03/30] added albumentations to poetry.lock --- poetry.lock | 412 ++++++++++++++++++++++++++-------------------------- 1 file changed, 208 insertions(+), 204 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6d8b9a3f0..b756ceea1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -198,46 +198,46 @@ files = [ [[package]] name = "connected-components-3d" -version = "3.12.2" +version = "3.12.3" description = "Connected components on 2D and 3D images. Supports multiple labels." optional = true python-versions = ">=3.7,<4.0" files = [ - {file = "connected-components-3d-3.12.2.tar.gz", hash = "sha256:867e9389b918db4b209ae754235d5f13bc276f131e36d9483159561479cb0582"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fc209503aa99a1fa03d02cb651246fa6f8d8a8675eff12046ff305d2e152fd41"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fc1d47df4b583631c23d5e900481f6c29e91481c5976fe8ab62b7f322b23bdea"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20b1c4bc3134c208dfc3e98adadf1e516784b63072533145956575d990f18ac9"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4dc9c1fc09cb213cf4f7bde6ba0dde7f47002ac78f2a590c1162aeaabf04ecd"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58a6bb8bc8c44ebe2e847236fa8106859e4de99dc5d79e44cc3a740563399916"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-win32.whl", hash = "sha256:bca4637c6a2a3bc3af146cfea0635fc742dcd65cfb4fc2100fbdce99c6ccadae"}, - {file = "connected_components_3d-3.12.2-cp310-cp310-win_amd64.whl", hash = "sha256:b4e89b15fd9bc80bd42b1a8d07f6f88d88915208c9b694453d0fa936e79a2acf"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56a7798c5dd14f49b67ecc1680cacf63a04cf196e6f48b0e8a6a9b20bb62c05b"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66ecc34575320fb2eb037a0e830b99c59ce5ddaecf8f14d7e237b1fbfc926f3d"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2f3ba7f41e240baa1eb33030d45c2a060ca94b3eb90e506bd6dc9e30a04e5ed"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd2a148a1d0fe902a5f4d437c61a2fb9eed023f6c6240c0242c6a72ecb2842ce"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd6cc42fb491a1d25c873a0f2a30bca5a07e771fc0ce92119a9a865c79c48456"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-win32.whl", hash = "sha256:d92b865b53524e87d118770927c0de19d1cfb02c9f59e0077362ad1a1a69434c"}, - {file = "connected_components_3d-3.12.2-cp311-cp311-win_amd64.whl", hash = "sha256:a6744e3442e913cb5da067b50bfc8005b8f5ff9b543b65acbecd6024ac790664"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:72a3657c3cea2fc17cac663205850857f69be8d54387353392c570296b927a0c"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade1ed09b044cef22defd6efb565930e6a8cc2732eaf7177d0a577ec928c50a0"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d49f539d8a880dc9c168b3dfe69e6cc3e4750fe31facd4f5f0e4188105d1107e"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4646776a33e724a0784cdeb3d51ddc3261ee238ef3f8e91f0277138516dfbd12"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-win32.whl", hash = "sha256:cc48fd8b717e3807158a659bed2b567c115d9033e9be968d8961d215e9725a34"}, - {file = "connected_components_3d-3.12.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ed6c1122dfdf5f6e494eedc18143c1e3a96453bc2b41791301370ac232c6d10b"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c85802e9ec4eb0abda5c4362abeaf24db2d420bbddac14913024062d13cf515"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:609f627d49f306a5fb66cb6eb386a87e1dce6bac91289dc4fa581744f6fd4fc7"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16ef4954e325b552bd1a3b3c14a8a9a97970390fbdf66df431b1b9bb841da774"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb2f9aafa5b3d3932be894b4eda9959aec32b5bb8f359cca4f2ba5db83ea38f3"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac93ce49b532a223f3fc32213720978f715c434f598548d381591782af10a823"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-win32.whl", hash = "sha256:3438d8eecf13ffbcdab2d90f4633d2c9edd9f6cecdf25c5f733a7e33ddb6c95c"}, - {file = "connected_components_3d-3.12.2-cp38-cp38-win_amd64.whl", hash = "sha256:14f574ba93c74c8c9e9e5d6a318a48aeaf1a8298d7f68960685c59f09a42452e"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c4ea235f9422ae41a89b9c37f1f5b277afd56afa95f1ec617a7b56b9e530955f"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1af878e907a3fa9792e9d3a9e14b4b3207f71a3a5243e38fff7df6c4d425ccfb"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ac306aae6eab080d14861b18d9748b6924bb8d04024b6e819ddf3de733075d8"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25d1d11bf70b76bc61588a915faf9b8780d97e0c50f368392f859c69ec93693c"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daf46a0017f3b1ee49bed20d1e6ad6a8cbeb5b9e715d397a79ad04b54ae77619"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-win32.whl", hash = "sha256:b066920dae690846791135ce4f6af117a5dc8f449df4e49d32f7564e2e5e4cdd"}, - {file = "connected_components_3d-3.12.2-cp39-cp39-win_amd64.whl", hash = "sha256:b9660819e778cb674afd0bef9f1b80fd8afc27fe42697d07a1741cd1310f4586"}, + {file = "connected-components-3d-3.12.3.tar.gz", hash = "sha256:0040a009d86918c757d607986edea760feb782093ddfc1c20761c8a88cd087ec"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef86445ab99428102272028cc291af6eb332031287bf275600419f0acd397d15"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a9a2aa53976b36a68ee701f021adadfd645527f10372515970c0373f4c254667"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df624b758ae6d22a0a7c2687f4d009bc7b722d58526872c8062a868472d05d70"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7997a95b655cd5dfbf94a00863e62c6b48a3ce54d96de492434c1decbabd8056"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc3fe77d67d82ebf42ef76e334517ac2165869a3393e5e5d68d475d36d057192"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-win32.whl", hash = "sha256:1dcd2c47dcc308825a091a61ca2da54cbc8dd994a322ade7cbc622ff8c3c1da7"}, + {file = "connected_components_3d-3.12.3-cp310-cp310-win_amd64.whl", hash = "sha256:7d124d12cc12766baaf08c3828282106a2407cb064950fd9bdd227f018471420"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b7c076ef6ea778f1581b68f6b994bf926543249057f795159ccec8661570bed"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc3a434bbf1b8033ee6e57ea4f499131d43ec5ff14d9c64b3c6cb11dd23b62a4"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c02b4d758093a676519182f38c1f50d6e3c2f251a181ec89c5919736b710789"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:073a715e0e0f877762e44fe83d98e651d2b4d476bd7862fd02fd80a0150f4d0f"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57996a349a0747e7262e8f740da13522c81bd7d09f9a9a1ecab359ba0ff1d90"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-win32.whl", hash = "sha256:010872cf018d2331b9c69148e3389118200ee1a24e719a17747acd3bad99b465"}, + {file = "connected_components_3d-3.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:d8a0062b5f4e4c205db46698f262d9dd4946e3c4660d3709b9606c6f4bce1586"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b3ffc46c0c10252f235130cf44fb0f643611104111871ebbaf21db3992f23041"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dbdfceb3ad240cd025d494f504b73dd4da9ebe19c053ff2036f7a5530a772c1f"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524fa48f8affbfe5b29f9a48976c7d948a552bfc865cd1aa6dd032d22d158c3e"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a115275f89233c726d6b0d1e543dd438094b5826c4b8e674de84370ae0320ca"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-win32.whl", hash = "sha256:b70ac3a2334c39bb608c0eb1721dbe2634e345942fc0bc4e60855672d4350c21"}, + {file = "connected_components_3d-3.12.3-cp37-cp37m-win_amd64.whl", hash = "sha256:31018fb4c4e6c9ba072c3a9e95638bdf6fa7c3fcd892a719ed9162686e975404"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca3674d9d7a6bc9e2029a5038a0bff085ec05cff21f0766d0e6fa54a0e3372f5"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:cc8c37a8aef38115a612978fdd37b9c98564ed9c05d09f3031c117408e06deba"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eeb301764b3da77cec3e8a869c1dd937395cda4fb7e63f05b9a4ac231eb8dd5"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aca594facf672997ac66aad4e86465dbc1b248a989474c67a549a70312f924c2"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccdec1820268a60dba4a63ec298767b9462c8beac18a315f91802643e50cc3c1"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-win32.whl", hash = "sha256:fbacf89984e374d8ae554c8e8fc4e4fc19b81e9396ce515aa893db6832114ebe"}, + {file = "connected_components_3d-3.12.3-cp38-cp38-win_amd64.whl", hash = "sha256:bb1de1dd58ac1303dcfee6370138de8704381c2f85c55fe545760416355fcb20"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6ee61e8e4cd61f182b583726e6965d1e9f81746ac2623199ee4c044fd941b2c0"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e17dc81b234e72668de3cf448766bb03aa57cceabb9469756a13feef7c8d60d6"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf2fe091b9fd2a64884662afa118dad96dcab04f45a35ab48a1a756fcf1d168a"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:385a266b94d6ba8ad11deabaa7ea6db094746e0ae6509759cc0d632c15bc7e3b"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95cb6e9c5fe25052f9029aa6f6a99896671123343f3a7039fd7b4cc0e940bfed"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-win32.whl", hash = "sha256:49954222a8936085284048eb0976e3916ae655505831fba2f80a5aa17c4cabaa"}, + {file = "connected_components_3d-3.12.3-cp39-cp39-win_amd64.whl", hash = "sha256:d8734fbbbd7da6ea960a0effdeed9263dc1dfbbfdb6134928d0ac8630d5b3a3b"}, ] [package.dependencies] @@ -245,29 +245,33 @@ numpy = "*" [[package]] name = "debugpy" -version = "1.6.7.post1" +version = "1.7.0" description = "An implementation of the Debug Adapter Protocol for Python" optional = true python-versions = ">=3.7" files = [ - {file = "debugpy-1.6.7.post1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:903bd61d5eb433b6c25b48eae5e23821d4c1a19e25c9610205f5aeaccae64e32"}, - {file = "debugpy-1.6.7.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16882030860081e7dd5aa619f30dec3c2f9a421e69861125f83cc372c94e57d"}, - {file = "debugpy-1.6.7.post1-cp310-cp310-win32.whl", hash = "sha256:eea8d8cfb9965ac41b99a61f8e755a8f50e9a20330938ad8271530210f54e09c"}, - {file = "debugpy-1.6.7.post1-cp310-cp310-win_amd64.whl", hash = "sha256:85969d864c45f70c3996067cfa76a319bae749b04171f2cdeceebe4add316155"}, - {file = "debugpy-1.6.7.post1-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:890f7ab9a683886a0f185786ffbda3b46495c4b929dab083b8c79d6825832a52"}, - {file = "debugpy-1.6.7.post1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4ac7a4dba28801d184b7fc0e024da2635ca87d8b0a825c6087bb5168e3c0d28"}, - {file = "debugpy-1.6.7.post1-cp37-cp37m-win32.whl", hash = "sha256:3370ef1b9951d15799ef7af41f8174194f3482ee689988379763ef61a5456426"}, - {file = "debugpy-1.6.7.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:65b28435a17cba4c09e739621173ff90c515f7b9e8ea469b92e3c28ef8e5cdfb"}, - {file = "debugpy-1.6.7.post1-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:92b6dae8bfbd497c90596bbb69089acf7954164aea3228a99d7e43e5267f5b36"}, - {file = "debugpy-1.6.7.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72f5d2ecead8125cf669e62784ef1e6300f4067b0f14d9f95ee00ae06fc7c4f7"}, - {file = "debugpy-1.6.7.post1-cp38-cp38-win32.whl", hash = "sha256:f0851403030f3975d6e2eaa4abf73232ab90b98f041e3c09ba33be2beda43fcf"}, - {file = "debugpy-1.6.7.post1-cp38-cp38-win_amd64.whl", hash = "sha256:3de5d0f97c425dc49bce4293df6a04494309eedadd2b52c22e58d95107e178d9"}, - {file = "debugpy-1.6.7.post1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:38651c3639a4e8bbf0ca7e52d799f6abd07d622a193c406be375da4d510d968d"}, - {file = "debugpy-1.6.7.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:038c51268367c9c935905a90b1c2d2dbfe304037c27ba9d19fe7409f8cdc710c"}, - {file = "debugpy-1.6.7.post1-cp39-cp39-win32.whl", hash = "sha256:4b9eba71c290852f959d2cf8a03af28afd3ca639ad374d393d53d367f7f685b2"}, - {file = "debugpy-1.6.7.post1-cp39-cp39-win_amd64.whl", hash = "sha256:973a97ed3b434eab0f792719a484566c35328196540676685c975651266fccf9"}, - {file = "debugpy-1.6.7.post1-py2.py3-none-any.whl", hash = "sha256:1093a5c541af079c13ac8c70ab8b24d1d35c8cacb676306cf11e57f699c02926"}, - {file = "debugpy-1.6.7.post1.zip", hash = "sha256:fe87ec0182ef624855d05e6ed7e0b7cb1359d2ffa2a925f8ec2d22e98b75d0ca"}, + {file = "debugpy-1.7.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:17ad9a681aca1704c55b9a5edcb495fa8f599e4655c9872b7f9cf3dc25890d48"}, + {file = "debugpy-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1285920a3f9a75f5d1acf59ab1b9da9ae6eb9a05884cd7674f95170c9cafa4de"}, + {file = "debugpy-1.7.0-cp310-cp310-win32.whl", hash = "sha256:a6f43a681c5025db1f1c0568069d1d1bad306a02e7c36144912b26d9c90e4724"}, + {file = "debugpy-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:9e9571d831ad3c75b5fb6f3efcb71c471cf2a74ba84af6ac1c79ce00683bed4b"}, + {file = "debugpy-1.7.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:538765a41198aa88cc089295b39c7322dd598f9ef1d52eaae12145c63bf9430a"}, + {file = "debugpy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7e8cf91f8f3f9b5fad844dd88427b85d398bda1e2a0cd65d5a21312fcbc0c6f"}, + {file = "debugpy-1.7.0-cp311-cp311-win32.whl", hash = "sha256:18a69f8e142a716310dd0af6d7db08992aed99e2606108732efde101e7c65e2a"}, + {file = "debugpy-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:7515a5ba5ee9bfe956685909c5f28734c1cecd4ee813523363acfe3ca824883a"}, + {file = "debugpy-1.7.0-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:bc8da67ade39d9e75608cdb8601d07e63a4e85966e0572c981f14e2cf42bcdef"}, + {file = "debugpy-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5036e918c6ba8fc4c4f1fd0207d81db634431a02f0dc2ba51b12fd793c8c9de"}, + {file = "debugpy-1.7.0-cp37-cp37m-win32.whl", hash = "sha256:d5be95b3946a4d7b388e45068c7b75036ac5a610f41014aee6cafcd5506423ad"}, + {file = "debugpy-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0e90314a078d4e3f009520c8387aba8f74c3034645daa7a332a3d1bb81335756"}, + {file = "debugpy-1.7.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:1565fd904f9571c430adca597771255cff4f92171486fced6f765dcbdfc8ec8d"}, + {file = "debugpy-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6516f36a2e95b3be27f171f12b641e443863f4ad5255d0fdcea6ae0be29bb912"}, + {file = "debugpy-1.7.0-cp38-cp38-win32.whl", hash = "sha256:2b0e489613bc066051439df04c56777ec184b957d6810cb65f235083aef7a0dc"}, + {file = "debugpy-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:7bf0b4bbd841b2397b6a8de15da9227f1164f6d43ceee971c50194eaed930a9d"}, + {file = "debugpy-1.7.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:ad22e1095b9977af432465c1e09132ba176e18df3834b1efcab1a449346b350b"}, + {file = "debugpy-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f625e427f21423e5874139db529e18cb2966bdfcc1cb87a195538c5b34d163d1"}, + {file = "debugpy-1.7.0-cp39-cp39-win32.whl", hash = "sha256:18bca8429d6632e2d3435055416d2d88f0309cc39709f4f6355c8d412cc61f24"}, + {file = "debugpy-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:dc8a12ac8b97ef3d6973c6679a093138c7c9b03eb685f0e253269a195f651559"}, + {file = "debugpy-1.7.0-py2.py3-none-any.whl", hash = "sha256:f6de2e6f24f62969e0f0ef682d78c98161c4dca29e9fb05df4d2989005005502"}, + {file = "debugpy-1.7.0.zip", hash = "sha256:676911c710e85567b17172db934a71319ed9d995104610ce23fd74a07f66e6f6"}, ] [[package]] @@ -723,18 +727,18 @@ wheel = "*" [[package]] name = "opencv-python-headless" -version = "4.8.0.76" +version = "4.8.1.78" description = "Wrapper package for OpenCV python bindings." optional = true python-versions = ">=3.6" files = [ - {file = "opencv-python-headless-4.8.0.76.tar.gz", hash = "sha256:bc15726187dae26d8a08777faf6bc71d38f20c785c102677f58ba0e935003afb"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-macosx_10_16_x86_64.whl", hash = "sha256:f85d2e3b9d952db35d31f9db8882d073c903921b72b8db1cfed8bbc75e8d3e63"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:8ee3bf1c9086493c340c6a87899f1c7778d729de92bce8560b8c31ab8a9cdf79"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c675b8dec6298ba6a1eec2ce24077a393b4236a043f68dfacb06bf594354ce06"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:220d2e292fa45ef0582aab730460bbc15cfe61f2089208167a372ccf76f01e21"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-win32.whl", hash = "sha256:df0608de207ae9b094ad9eaf1a475cf6e9a069fb12cd289d4a18cefdab2f8aa8"}, - {file = "opencv_python_headless-4.8.0.76-cp37-abi3-win_amd64.whl", hash = "sha256:9c094faf6ec7bd360244647b26ebdf8f54edec1d9292cb9179fff9badcca7be8"}, + {file = "opencv-python-headless-4.8.1.78.tar.gz", hash = "sha256:bc7197b42352f6f865c302a49140b889ec7cd957dd697e2d7fc016ad0d3f28f1"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-macosx_10_16_x86_64.whl", hash = "sha256:f3a33f644249f9ce1c913eac580e4b3ef4ce7cab0a71900274708959c2feb5e3"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:2c7d45721df9801c4dcd34683a15caa0e30f38b185263fec04a6eb274bc720f0"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b6bd6e1132b6f5dcb3a5bfe30fc4d341a7bfb26134da349a06c9255288ded94"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58e70d2f0915fe23e02c6e405588276c9397844a47d38b9c87fac5f7f9ba2dcc"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-win32.whl", hash = "sha256:382f8c7a6a14f80091284eecedd52cee4812231ee0eff1118592197b538d9252"}, + {file = "opencv_python_headless-4.8.1.78-cp37-abi3-win_amd64.whl", hash = "sha256:0a0f1e9f836f7d5bad1dd164694944c8761711cbdf4b36ebbd4815a8ef731079"}, ] [package.dependencies] @@ -749,71 +753,71 @@ numpy = [ [[package]] name = "orjson" -version = "3.9.5" +version = "3.9.7" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false python-versions = ">=3.7" files = [ - {file = "orjson-3.9.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ad6845912a71adcc65df7c8a7f2155eba2096cf03ad2c061c93857de70d699ad"}, - {file = "orjson-3.9.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e298e0aacfcc14ef4476c3f409e85475031de24e5b23605a465e9bf4b2156273"}, - {file = "orjson-3.9.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:83c9939073281ef7dd7c5ca7f54cceccb840b440cec4b8a326bda507ff88a0a6"}, - {file = "orjson-3.9.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e174cc579904a48ee1ea3acb7045e8a6c5d52c17688dfcb00e0e842ec378cabf"}, - {file = "orjson-3.9.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f8d51702f42c785b115401e1d64a27a2ea767ae7cf1fb8edaa09c7cf1571c660"}, - {file = "orjson-3.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13d61c0c7414ddee1ef4d0f303e2222f8cced5a2e26d9774751aecd72324c9e"}, - {file = "orjson-3.9.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d748cc48caf5a91c883d306ab648df1b29e16b488c9316852844dd0fd000d1c2"}, - {file = "orjson-3.9.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bd19bc08fa023e4c2cbf8294ad3f2b8922f4de9ba088dbc71e6b268fdf54591c"}, - {file = "orjson-3.9.5-cp310-none-win32.whl", hash = "sha256:5793a21a21bf34e1767e3d61a778a25feea8476dcc0bdf0ae1bc506dc34561ea"}, - {file = "orjson-3.9.5-cp310-none-win_amd64.whl", hash = "sha256:2bcec0b1024d0031ab3eab7a8cb260c8a4e4a5e35993878a2da639d69cdf6a65"}, - {file = "orjson-3.9.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8547b95ca0e2abd17e1471973e6d676f1d8acedd5f8fb4f739e0612651602d66"}, - {file = "orjson-3.9.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87ce174d6a38d12b3327f76145acbd26f7bc808b2b458f61e94d83cd0ebb4d76"}, - {file = "orjson-3.9.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a960bb1bc9a964d16fcc2d4af5a04ce5e4dfddca84e3060c35720d0a062064fe"}, - {file = "orjson-3.9.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a7aa5573a949760d6161d826d34dc36db6011926f836851fe9ccb55b5a7d8e8"}, - {file = "orjson-3.9.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b2852afca17d7eea85f8e200d324e38c851c96598ac7b227e4f6c4e59fbd3df"}, - {file = "orjson-3.9.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa185959c082475288da90f996a82e05e0c437216b96f2a8111caeb1d54ef926"}, - {file = "orjson-3.9.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:89c9332695b838438ea4b9a482bce8ffbfddde4df92750522d928fb00b7b8dce"}, - {file = "orjson-3.9.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2493f1351a8f0611bc26e2d3d407efb873032b4f6b8926fed8cfed39210ca4ba"}, - {file = "orjson-3.9.5-cp311-none-win32.whl", hash = "sha256:ffc544e0e24e9ae69301b9a79df87a971fa5d1c20a6b18dca885699709d01be0"}, - {file = "orjson-3.9.5-cp311-none-win_amd64.whl", hash = "sha256:89670fe2732e3c0c54406f77cad1765c4c582f67b915c74fda742286809a0cdc"}, - {file = "orjson-3.9.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:15df211469625fa27eced4aa08dc03e35f99c57d45a33855cc35f218ea4071b8"}, - {file = "orjson-3.9.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9f17c59fe6c02bc5f89ad29edb0253d3059fe8ba64806d789af89a45c35269a"}, - {file = "orjson-3.9.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca6b96659c7690773d8cebb6115c631f4a259a611788463e9c41e74fa53bf33f"}, - {file = "orjson-3.9.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26fafe966e9195b149950334bdbe9026eca17fe8ffe2d8fa87fdc30ca925d30"}, - {file = "orjson-3.9.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9006b1eb645ecf460da067e2dd17768ccbb8f39b01815a571bfcfab7e8da5e52"}, - {file = "orjson-3.9.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfdbf695734b1785e792a1315e41835ddf2a3e907ca0e1c87a53f23006ce01d"}, - {file = "orjson-3.9.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4a3943234342ab37d9ed78fb0a8f81cd4b9532f67bf2ac0d3aa45fa3f0a339f3"}, - {file = "orjson-3.9.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e6762755470b5c82f07b96b934af32e4d77395a11768b964aaa5eb092817bc31"}, - {file = "orjson-3.9.5-cp312-none-win_amd64.whl", hash = "sha256:c74df28749c076fd6e2157190df23d43d42b2c83e09d79b51694ee7315374ad5"}, - {file = "orjson-3.9.5-cp37-cp37m-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:88e18a74d916b74f00d0978d84e365c6bf0e7ab846792efa15756b5fb2f7d49d"}, - {file = "orjson-3.9.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d28514b5b6dfaf69097be70d0cf4f1407ec29d0f93e0b4131bf9cc8fd3f3e374"}, - {file = "orjson-3.9.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b81aca8c7be61e2566246b6a0ca49f8aece70dd3f38c7f5c837f398c4cb142"}, - {file = "orjson-3.9.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:385c1c713b1e47fd92e96cf55fd88650ac6dfa0b997e8aa7ecffd8b5865078b1"}, - {file = "orjson-3.9.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f9850c03a8e42fba1a508466e6a0f99472fd2b4a5f30235ea49b2a1b32c04c11"}, - {file = "orjson-3.9.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4449f84bbb13bcef493d8aa669feadfced0f7c5eea2d0d88b5cc21f812183af8"}, - {file = "orjson-3.9.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:86127bf194f3b873135e44ce5dc9212cb152b7e06798d5667a898a00f0519be4"}, - {file = "orjson-3.9.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0abcd039f05ae9ab5b0ff11624d0b9e54376253b7d3217a358d09c3edf1d36f7"}, - {file = "orjson-3.9.5-cp37-none-win32.whl", hash = "sha256:10cc8ad5ff7188efcb4bec196009d61ce525a4e09488e6d5db41218c7fe4f001"}, - {file = "orjson-3.9.5-cp37-none-win_amd64.whl", hash = "sha256:ff27e98532cb87379d1a585837d59b187907228268e7b0a87abe122b2be6968e"}, - {file = "orjson-3.9.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5bfa79916ef5fef75ad1f377e54a167f0de334c1fa4ebb8d0224075f3ec3d8c0"}, - {file = "orjson-3.9.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e87dfa6ac0dae764371ab19b35eaaa46dfcb6ef2545dfca03064f21f5d08239f"}, - {file = "orjson-3.9.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50ced24a7b23058b469ecdb96e36607fc611cbaee38b58e62a55c80d1b3ad4e1"}, - {file = "orjson-3.9.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1b74ea2a3064e1375da87788897935832e806cc784de3e789fd3c4ab8eb3fa5"}, - {file = "orjson-3.9.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7cb961efe013606913d05609f014ad43edfaced82a576e8b520a5574ce3b2b9"}, - {file = "orjson-3.9.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1225d2d5ee76a786bda02f8c5e15017462f8432bb960de13d7c2619dba6f0275"}, - {file = "orjson-3.9.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f39f4b99199df05c7ecdd006086259ed25886cdbd7b14c8cdb10c7675cfcca7d"}, - {file = "orjson-3.9.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a461dc9fb60cac44f2d3218c36a0c1c01132314839a0e229d7fb1bba69b810d8"}, - {file = "orjson-3.9.5-cp38-none-win32.whl", hash = "sha256:dedf1a6173748202df223aea29de814b5836732a176b33501375c66f6ab7d822"}, - {file = "orjson-3.9.5-cp38-none-win_amd64.whl", hash = "sha256:fa504082f53efcbacb9087cc8676c163237beb6e999d43e72acb4bb6f0db11e6"}, - {file = "orjson-3.9.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6900f0248edc1bec2a2a3095a78a7e3ef4e63f60f8ddc583687eed162eedfd69"}, - {file = "orjson-3.9.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17404333c40047888ac40bd8c4d49752a787e0a946e728a4e5723f111b6e55a5"}, - {file = "orjson-3.9.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0eefb7cfdd9c2bc65f19f974a5d1dfecbac711dae91ed635820c6b12da7a3c11"}, - {file = "orjson-3.9.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:68c78b2a3718892dc018adbc62e8bab6ef3c0d811816d21e6973dee0ca30c152"}, - {file = "orjson-3.9.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:591ad7d9e4a9f9b104486ad5d88658c79ba29b66c5557ef9edf8ca877a3f8d11"}, - {file = "orjson-3.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cc2cbf302fbb2d0b2c3c142a663d028873232a434d89ce1b2604ebe5cc93ce8"}, - {file = "orjson-3.9.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b26b5aa5e9ee1bad2795b925b3adb1b1b34122cb977f30d89e0a1b3f24d18450"}, - {file = "orjson-3.9.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ef84724f7d29dcfe3aafb1fc5fc7788dca63e8ae626bb9298022866146091a3e"}, - {file = "orjson-3.9.5-cp39-none-win32.whl", hash = "sha256:664cff27f85939059472afd39acff152fbac9a091b7137092cb651cf5f7747b5"}, - {file = "orjson-3.9.5-cp39-none-win_amd64.whl", hash = "sha256:91dda66755795ac6100e303e206b636568d42ac83c156547634256a2e68de694"}, - {file = "orjson-3.9.5.tar.gz", hash = "sha256:6daf5ee0b3cf530b9978cdbf71024f1c16ed4a67d05f6ec435c6e7fe7a52724c"}, + {file = "orjson-3.9.7-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6df858e37c321cefbf27fe7ece30a950bcc3a75618a804a0dcef7ed9dd9c92d"}, + {file = "orjson-3.9.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5198633137780d78b86bb54dafaaa9baea698b4f059456cd4554ab7009619221"}, + {file = "orjson-3.9.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e736815b30f7e3c9044ec06a98ee59e217a833227e10eb157f44071faddd7c5"}, + {file = "orjson-3.9.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a19e4074bc98793458b4b3ba35a9a1d132179345e60e152a1bb48c538ab863c4"}, + {file = "orjson-3.9.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80acafe396ab689a326ab0d80f8cc61dec0dd2c5dca5b4b3825e7b1e0132c101"}, + {file = "orjson-3.9.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:355efdbbf0cecc3bd9b12589b8f8e9f03c813a115efa53f8dc2a523bfdb01334"}, + {file = "orjson-3.9.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3aab72d2cef7f1dd6104c89b0b4d6b416b0db5ca87cc2fac5f79c5601f549cc2"}, + {file = "orjson-3.9.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:36b1df2e4095368ee388190687cb1b8557c67bc38400a942a1a77713580b50ae"}, + {file = "orjson-3.9.7-cp310-none-win32.whl", hash = "sha256:e94b7b31aa0d65f5b7c72dd8f8227dbd3e30354b99e7a9af096d967a77f2a580"}, + {file = "orjson-3.9.7-cp310-none-win_amd64.whl", hash = "sha256:82720ab0cf5bb436bbd97a319ac529aee06077ff7e61cab57cee04a596c4f9b4"}, + {file = "orjson-3.9.7-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1f8b47650f90e298b78ecf4df003f66f54acdba6a0f763cc4df1eab048fe3738"}, + {file = "orjson-3.9.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f738fee63eb263530efd4d2e9c76316c1f47b3bbf38c1bf45ae9625feed0395e"}, + {file = "orjson-3.9.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38e34c3a21ed41a7dbd5349e24c3725be5416641fdeedf8f56fcbab6d981c900"}, + {file = "orjson-3.9.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21a3344163be3b2c7e22cef14fa5abe957a892b2ea0525ee86ad8186921b6cf0"}, + {file = "orjson-3.9.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23be6b22aab83f440b62a6f5975bcabeecb672bc627face6a83bc7aeb495dc7e"}, + {file = "orjson-3.9.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5205ec0dfab1887dd383597012199f5175035e782cdb013c542187d280ca443"}, + {file = "orjson-3.9.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8769806ea0b45d7bf75cad253fba9ac6700b7050ebb19337ff6b4e9060f963fa"}, + {file = "orjson-3.9.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f9e01239abea2f52a429fe9d95c96df95f078f0172489d691b4a848ace54a476"}, + {file = "orjson-3.9.7-cp311-none-win32.whl", hash = "sha256:8bdb6c911dae5fbf110fe4f5cba578437526334df381b3554b6ab7f626e5eeca"}, + {file = "orjson-3.9.7-cp311-none-win_amd64.whl", hash = "sha256:9d62c583b5110e6a5cf5169ab616aa4ec71f2c0c30f833306f9e378cf51b6c86"}, + {file = "orjson-3.9.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1c3cee5c23979deb8d1b82dc4cc49be59cccc0547999dbe9adb434bb7af11cf7"}, + {file = "orjson-3.9.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a347d7b43cb609e780ff8d7b3107d4bcb5b6fd09c2702aa7bdf52f15ed09fa09"}, + {file = "orjson-3.9.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:154fd67216c2ca38a2edb4089584504fbb6c0694b518b9020ad35ecc97252bb9"}, + {file = "orjson-3.9.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ea3e63e61b4b0beeb08508458bdff2daca7a321468d3c4b320a758a2f554d31"}, + {file = "orjson-3.9.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1eb0b0b2476f357eb2975ff040ef23978137aa674cd86204cfd15d2d17318588"}, + {file = "orjson-3.9.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b9a20a03576c6b7022926f614ac5a6b0914486825eac89196adf3267c6489d"}, + {file = "orjson-3.9.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:915e22c93e7b7b636240c5a79da5f6e4e84988d699656c8e27f2ac4c95b8dcc0"}, + {file = "orjson-3.9.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f26fb3e8e3e2ee405c947ff44a3e384e8fa1843bc35830fe6f3d9a95a1147b6e"}, + {file = "orjson-3.9.7-cp312-none-win_amd64.whl", hash = "sha256:d8692948cada6ee21f33db5e23460f71c8010d6dfcfe293c9b96737600a7df78"}, + {file = "orjson-3.9.7-cp37-cp37m-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7bab596678d29ad969a524823c4e828929a90c09e91cc438e0ad79b37ce41166"}, + {file = "orjson-3.9.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63ef3d371ea0b7239ace284cab9cd00d9c92b73119a7c274b437adb09bda35e6"}, + {file = "orjson-3.9.7-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f8fcf696bbbc584c0c7ed4adb92fd2ad7d153a50258842787bc1524e50d7081"}, + {file = "orjson-3.9.7-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:90fe73a1f0321265126cbba13677dcceb367d926c7a65807bd80916af4c17047"}, + {file = "orjson-3.9.7-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45a47f41b6c3beeb31ac5cf0ff7524987cfcce0a10c43156eb3ee8d92d92bf22"}, + {file = "orjson-3.9.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a2937f528c84e64be20cb80e70cea76a6dfb74b628a04dab130679d4454395c"}, + {file = "orjson-3.9.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b4fb306c96e04c5863d52ba8d65137917a3d999059c11e659eba7b75a69167bd"}, + {file = "orjson-3.9.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:410aa9d34ad1089898f3db461b7b744d0efcf9252a9415bbdf23540d4f67589f"}, + {file = "orjson-3.9.7-cp37-none-win32.whl", hash = "sha256:26ffb398de58247ff7bde895fe30817a036f967b0ad0e1cf2b54bda5f8dcfdd9"}, + {file = "orjson-3.9.7-cp37-none-win_amd64.whl", hash = "sha256:bcb9a60ed2101af2af450318cd89c6b8313e9f8df4e8fb12b657b2e97227cf08"}, + {file = "orjson-3.9.7-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5da9032dac184b2ae2da4bce423edff7db34bfd936ebd7d4207ea45840f03905"}, + {file = "orjson-3.9.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7951af8f2998045c656ba8062e8edf5e83fd82b912534ab1de1345de08a41d2b"}, + {file = "orjson-3.9.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b8e59650292aa3a8ea78073fc84184538783966528e442a1b9ed653aa282edcf"}, + {file = "orjson-3.9.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9274ba499e7dfb8a651ee876d80386b481336d3868cba29af839370514e4dce0"}, + {file = "orjson-3.9.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca1706e8b8b565e934c142db6a9592e6401dc430e4b067a97781a997070c5378"}, + {file = "orjson-3.9.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83cc275cf6dcb1a248e1876cdefd3f9b5f01063854acdfd687ec360cd3c9712a"}, + {file = "orjson-3.9.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:11c10f31f2c2056585f89d8229a56013bc2fe5de51e095ebc71868d070a8dd81"}, + {file = "orjson-3.9.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cf334ce1d2fadd1bf3e5e9bf15e58e0c42b26eb6590875ce65bd877d917a58aa"}, + {file = "orjson-3.9.7-cp38-none-win32.whl", hash = "sha256:76a0fc023910d8a8ab64daed8d31d608446d2d77c6474b616b34537aa7b79c7f"}, + {file = "orjson-3.9.7-cp38-none-win_amd64.whl", hash = "sha256:7a34a199d89d82d1897fd4a47820eb50947eec9cda5fd73f4578ff692a912f89"}, + {file = "orjson-3.9.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e7e7f44e091b93eb39db88bb0cb765db09b7a7f64aea2f35e7d86cbf47046c65"}, + {file = "orjson-3.9.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d647b2a9c45a23a84c3e70e19d120011cba5f56131d185c1b78685457320bb"}, + {file = "orjson-3.9.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0eb850a87e900a9c484150c414e21af53a6125a13f6e378cf4cc11ae86c8f9c5"}, + {file = "orjson-3.9.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f4b0042d8388ac85b8330b65406c84c3229420a05068445c13ca28cc222f1f7"}, + {file = "orjson-3.9.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd3e7aae977c723cc1dbb82f97babdb5e5fbce109630fbabb2ea5053523c89d3"}, + {file = "orjson-3.9.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c616b796358a70b1f675a24628e4823b67d9e376df2703e893da58247458956"}, + {file = "orjson-3.9.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c3ba725cf5cf87d2d2d988d39c6a2a8b6fc983d78ff71bc728b0be54c869c884"}, + {file = "orjson-3.9.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4891d4c934f88b6c29b56395dfc7014ebf7e10b9e22ffd9877784e16c6b2064f"}, + {file = "orjson-3.9.7-cp39-none-win32.whl", hash = "sha256:14d3fb6cd1040a4a4a530b28e8085131ed94ebc90d72793c59a713de34b60838"}, + {file = "orjson-3.9.7-cp39-none-win_amd64.whl", hash = "sha256:9ef82157bbcecd75d6296d5d8b2d792242afcd064eb1ac573f8847b52e58f677"}, + {file = "orjson-3.9.7.tar.gz", hash = "sha256:85e39198f78e2f7e054d296395f6c96f5e02892337746ef5b6a1bf3ed5910142"}, ] [[package]] @@ -977,47 +981,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.12" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, - {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, - {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, - {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, - {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, - {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, - {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, - {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, - {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, - {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] @@ -1090,13 +1094,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = true python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -1251,13 +1255,13 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.5.2" +version = "13.5.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, - {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, + {file = "rich-13.5.3-py3-none-any.whl", hash = "sha256:9257b468badc3d347e146a4faa268ff229039d4c2d176ab0cffb4c4fbc73d5d9"}, + {file = "rich-13.5.3.tar.gz", hash = "sha256:87b43e0543149efa1253f485cd845bb7ee54df16c9617b8a893650ab84b4acb6"}, ] [package.dependencies] @@ -1270,37 +1274,37 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "scikit-learn" -version = "1.3.0" +version = "1.3.1" description = "A set of python modules for machine learning and data mining" optional = true python-versions = ">=3.8" files = [ - {file = "scikit-learn-1.3.0.tar.gz", hash = "sha256:8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a"}, - {file = "scikit_learn-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:981287869e576d42c682cf7ca96af0c6ac544ed9316328fd0d9292795c742cf5"}, - {file = "scikit_learn-1.3.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:436aaaae2c916ad16631142488e4c82f4296af2404f480e031d866863425d2a2"}, - {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7e28d8fa47a0b30ae1bd7a079519dd852764e31708a7804da6cb6f8b36e3630"}, - {file = "scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae80c08834a473d08a204d966982a62e11c976228d306a2648c575e3ead12111"}, - {file = "scikit_learn-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:552fd1b6ee22900cf1780d7386a554bb96949e9a359999177cf30211e6b20df6"}, - {file = "scikit_learn-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79970a6d759eb00a62266a31e2637d07d2d28446fca8079cf9afa7c07b0427f8"}, - {file = "scikit_learn-1.3.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:850a00b559e636b23901aabbe79b73dc604b4e4248ba9e2d6e72f95063765603"}, - {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee04835fb016e8062ee9fe9074aef9b82e430504e420bff51e3e5fffe72750ca"}, - {file = "scikit_learn-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d953531f5d9f00c90c34fa3b7d7cfb43ecff4c605dac9e4255a20b114a27369"}, - {file = "scikit_learn-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:151ac2bf65ccf363664a689b8beafc9e6aae36263db114b4ca06fbbbf827444a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a885a9edc9c0a341cab27ec4f8a6c58b35f3d449c9d2503a6fd23e06bbd4f6a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:9877af9c6d1b15486e18a94101b742e9d0d2f343d35a634e337411ddb57783f3"}, - {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c470f53cea065ff3d588050955c492793bb50c19a92923490d18fcb637f6383a"}, - {file = "scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6e2d7389542eae01077a1ee0318c4fec20c66c957f45c7aac0c6eb0fe3c612"}, - {file = "scikit_learn-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a11936adbc379a6061ea32fa03338d4ca7248d86dd507c81e13af428a5bc1db"}, - {file = "scikit_learn-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:998d38fcec96584deee1e79cd127469b3ad6fefd1ea6c2dfc54e8db367eb396b"}, - {file = "scikit_learn-1.3.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ded35e810438a527e17623ac6deae3b360134345b7c598175ab7741720d7ffa7"}, - {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e8102d5036e28d08ab47166b48c8d5e5810704daecf3a476a4282d562be9a28"}, - {file = "scikit_learn-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7617164951c422747e7c32be4afa15d75ad8044f42e7d70d3e2e0429a50e6718"}, - {file = "scikit_learn-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:1d54fb9e6038284548072df22fd34777e434153f7ffac72c8596f2d6987110dd"}, + {file = "scikit-learn-1.3.1.tar.gz", hash = "sha256:1a231cced3ee3fa04756b4a7ab532dc9417acd581a330adff5f2c01ac2831fcf"}, + {file = "scikit_learn-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3153612ff8d36fa4e35ef8b897167119213698ea78f3fd130b4068e6f8d2da5a"}, + {file = "scikit_learn-1.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:6bb9490fdb8e7e00f1354621689187bef3cab289c9b869688f805bf724434755"}, + {file = "scikit_learn-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7135a03af71138669f19bc96e7d0cc8081aed4b3565cc3b131135d65fc642ba"}, + {file = "scikit_learn-1.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d8dee8c1f40eeba49a85fe378bdf70a07bb64aba1a08fda1e0f48d27edfc3e6"}, + {file = "scikit_learn-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:4d379f2b34096105a96bd857b88601dffe7389bd55750f6f29aaa37bc6272eb5"}, + {file = "scikit_learn-1.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14e8775eba072ab10866a7e0596bc9906873e22c4c370a651223372eb62de180"}, + {file = "scikit_learn-1.3.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:58b0c2490eff8355dc26e884487bf8edaccf2ba48d09b194fb2f3a026dd64f9d"}, + {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f66eddfda9d45dd6cadcd706b65669ce1df84b8549875691b1f403730bdef217"}, + {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6448c37741145b241eeac617028ba6ec2119e1339b1385c9720dae31367f2be"}, + {file = "scikit_learn-1.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c413c2c850241998168bbb3bd1bb59ff03b1195a53864f0b80ab092071af6028"}, + {file = "scikit_learn-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:52b77cc08bd555969ec5150788ed50276f5ef83abb72e6f469c5b91a0009bbca"}, + {file = "scikit_learn-1.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a683394bc3f80b7c312c27f9b14ebea7766b1f0a34faf1a2e9158d80e860ec26"}, + {file = "scikit_learn-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15d964d9eb181c79c190d3dbc2fff7338786bf017e9039571418a1d53dab236"}, + {file = "scikit_learn-1.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ce9233cdf0cdcf0858a5849d306490bf6de71fa7603a3835124e386e62f2311"}, + {file = "scikit_learn-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:1ec668ce003a5b3d12d020d2cde0abd64b262ac5f098b5c84cf9657deb9996a8"}, + {file = "scikit_learn-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccbbedae99325628c1d1cbe3916b7ef58a1ce949672d8d39c8b190e10219fd32"}, + {file = "scikit_learn-1.3.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:845f81c7ceb4ea6bac64ab1c9f2ce8bef0a84d0f21f3bece2126adcc213dfecd"}, + {file = "scikit_learn-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8454d57a22d856f1fbf3091bd86f9ebd4bff89088819886dc0c72f47a6c30652"}, + {file = "scikit_learn-1.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d993fb70a1d78c9798b8f2f28705bfbfcd546b661f9e2e67aa85f81052b9c53"}, + {file = "scikit_learn-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:66f7bb1fec37d65f4ef85953e1df5d3c98a0f0141d394dcdaead5a6de9170347"}, ] [package.dependencies] joblib = ">=1.1.1" -numpy = ">=1.17.3" +numpy = ">=1.17.3,<2.0" scipy = ">=1.5.0" threadpoolctl = ">=2.0.0" @@ -1547,24 +1551,24 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.11" +version = "6.0.12.12" description = "Typing stubs for PyYAML" optional = false python-versions = "*" files = [ - {file = "types-PyYAML-6.0.12.11.tar.gz", hash = "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b"}, - {file = "types_PyYAML-6.0.12.11-py3-none-any.whl", hash = "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d"}, + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, ] [[package]] name = "types-requests" -version = "2.31.0.2" +version = "2.31.0.6" description = "Typing stubs for requests" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "types-requests-2.31.0.2.tar.gz", hash = "sha256:6aa3f7faf0ea52d728bb18c0a0d1522d9bfd8c72d26ff6f61bfc3d06a411cf40"}, - {file = "types_requests-2.31.0.2-py3-none-any.whl", hash = "sha256:56d181c85b5925cbc59f4489a57e72a8b2166f18273fd8ba7b6fe0c0b986f12a"}, + {file = "types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0"}, + {file = "types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9"}, ] [package.dependencies] @@ -1626,13 +1630,13 @@ numpy = "*" [[package]] name = "urllib3" -version = "2.0.4" +version = "2.0.5" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, - {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, ] [package.extras] @@ -1643,13 +1647,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "wheel" -version = "0.41.1" +version = "0.41.2" description = "A built-package format for Python" optional = true python-versions = ">=3.7" files = [ - {file = "wheel-0.41.1-py3-none-any.whl", hash = "sha256:473219bd4cbedc62cea0cb309089b593e47c15c4a2531015f94e4e3b9a0f6981"}, - {file = "wheel-0.41.1.tar.gz", hash = "sha256:12b911f083e876e10c595779709f8a88a59f45aacc646492a67fe9ef796c1b47"}, + {file = "wheel-0.41.2-py3-none-any.whl", hash = "sha256:75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8"}, + {file = "wheel-0.41.2.tar.gz", hash = "sha256:0c5ac5ff2afb79ac23ab82bab027a0be7b5dbcf2e54dc50efe4bf507de1f7985"}, ] [package.extras] @@ -1673,7 +1677,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [extras] dev = ["black", "debugpy", "flake8", "flake8-pyproject", "isort", "mypy", "pytest", "responses"] medical = ["connected-components-3d", "nibabel"] -ml = ["scikit-learn", "torch", "torchvision"] +ml = ["scikit-learn", "torch", "torchvision", "albumentations"] ocv = ["opencv-python-headless"] test = ["flake8-pyproject", "pytest", "responses"] From 38fb230345113a62cb3cdc3873250abe11edfc62 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Fri, 29 Sep 2023 15:06:20 +0200 Subject: [PATCH 04/30] added manual install of albumentations --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 249d5b8cd..c31739f2f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,6 +62,7 @@ jobs: pip install wheel pip install --upgrade setuptools pip install --editable ".[test,ml,medical,dev,ocv]" + pip install albumentations pip install pytest - name: Run Tests run: python -m pytest -W ignore::DeprecationWarning From 9e83781914750ee7792c80aa0ed0ee25815df8fd Mon Sep 17 00:00:00 2001 From: Christoffer Date: Mon, 9 Oct 2023 14:22:29 +0200 Subject: [PATCH 05/30] added support to load both polygon and bounding-box annotations for object-detection datasets, also added better error msg --- darwin/dataset/utils.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 3f3bb865f..7209c50b2 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -131,6 +131,19 @@ def make_class_lists(release_path: Path) -> None: f.write("\n".join(classes_names)) +def get_classes_from_file(path): + """Helper function to read class names from a file.""" + if path.exists(): + return path.read_text().splitlines() + return [] + + +def available_annotation_types(release_path): + """Returns a list of available annotation types based on the existing files.""" + files = [p.name for p in release_path.glob("lists/classes_*.txt")] + return [f[len("classes_") : -len(".txt")] for f in files] + + def get_classes( dataset_path: PathLike, release_name: Optional[str] = None, @@ -147,7 +160,7 @@ def get_classes( release_name : Optional[str], default: None Version of the dataset. annotation_type : str, default: "polygon" - The type of annotation classes [tag, polygon]. + The type of annotation classes [tag, polygon, bounding_box]. remove_background : bool, default: True Removes the background class (if exists) from the list of classes. @@ -160,10 +173,24 @@ def get_classes( dataset_path = Path(dataset_path) release_path = get_release_path(dataset_path, release_name) - classes_path = release_path / f"lists/classes_{annotation_type}.txt" - classes = classes_path.read_text().splitlines() - if remove_background and classes[0] == "__background__": + # If annotation_type is a string and is 'bounding_box', also consider polygons + if isinstance(annotation_type, str): + if annotation_type == "bounding_box": + annotation_types_to_load = [annotation_type, "polygon"] + else: + annotation_types_to_load = [annotation_type] + + classes = [] + for atype in annotation_types_to_load: + classes_file_path = release_path / f"lists/classes_{atype}.txt" + classes.extend(get_classes_from_file(classes_file_path)) + + if remove_background and classes and classes[0] == "__background__": classes = classes[1:] + + available_types = available_annotation_types(release_path) + assert len(classes) > 0, f"No classes found for {annotation_type}. Supported types are: {', '.join(available_types)}" + return classes @@ -627,9 +654,7 @@ def compute_distributions( if annotation_file is None: continue - annotation_class_names: List[str] = [ - annotation.annotation_class.name for annotation in annotation_file.annotations - ] + annotation_class_names: List[str] = [annotation.annotation_class.name for annotation in annotation_file.annotations] class_distribution[partition] += Counter(set(annotation_class_names)) instance_distribution[partition] += Counter(annotation_class_names) From 264fe333e943b346bacdb83b6136c82ee6649e12 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Mon, 9 Oct 2023 14:59:55 +0200 Subject: [PATCH 06/30] commit --- tests/darwin/torch/dataset_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index 50484c00c..5593dd4ca 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -74,6 +74,7 @@ def test_should_correctly_create_a_object_detection_dataset( self, team_slug: str, team_extracted_dataset_path: Path ) -> None: root = team_extracted_dataset_path / team_slug / "coco" + print(root) ds = ObjectDetectionDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) From 31dc64c1c26d5b4eb6d6e38702f9887d511c6544 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Mon, 9 Oct 2023 15:13:01 +0200 Subject: [PATCH 07/30] removed test that will be introduced in another pr --- tests/darwin/torch/transform_test.py | 84 ---------------------------- 1 file changed, 84 deletions(-) delete mode 100644 tests/darwin/torch/transform_test.py diff --git a/tests/darwin/torch/transform_test.py b/tests/darwin/torch/transform_test.py deleted file mode 100644 index 8bc974b08..000000000 --- a/tests/darwin/torch/transform_test.py +++ /dev/null @@ -1,84 +0,0 @@ -import numpy as np -import pytest -import torch -from albumentations import BboxParams, Compose, HorizontalFlip, Resize -from PIL import Image - -from darwin.torch.transforms import AlbumentationsTransform - -# Sample data -SAMPLE_IMAGE = Image.new("RGB", (100, 100)) -SAMPLE_ANNOTATION = {"boxes": torch.tensor([[25, 25, 75, 75]]), "labels": torch.tensor([1]), "area": torch.tensor([2500.0]), "iscrowd": torch.tensor([0])} - -SAMPLE_ANNOTATION_OOB = { - "boxes": torch.tensor([[25, 25, 105, 105]]), # Out of bounds - "labels": torch.tensor([1]), - "area": torch.tensor([2500.0]), - "iscrowd": torch.tensor([0]), -} - -SAMPLE_ANNOTATION_WITH_MASKS = {**SAMPLE_ANNOTATION, "masks": torch.ones((1, 100, 100))} - -EXAMPLE_TRANSFORM = Compose([HorizontalFlip(p=1)], bbox_params=BboxParams(format="coco", label_fields=["labels"])) -EXAMPLE_TRANSFORM_RESIZE = Compose([Resize(50, 50)], bbox_params=BboxParams(format="coco", label_fields=["labels"])) - - -class TestAlbumentationsTransform: - def test_init(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - assert isinstance(at, AlbumentationsTransform) - - def test_from_path_invalid(self): - with pytest.raises(ValueError): - AlbumentationsTransform.from_path("invalid/path/to/config.yml") - - def test_from_dict_invalid(self): - with pytest.raises(ValueError): - AlbumentationsTransform.from_dict({"invalid": "config"}) - - def test_transformations(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - image, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) - assert annotation["boxes"][0, 0] != SAMPLE_ANNOTATION["boxes"][0, 0] - - def test_transformations_resize(self): - transformations = EXAMPLE_TRANSFORM_RESIZE - at = AlbumentationsTransform(transformations) - image, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) - assert image.shape[:2] == (50, 50) # We only check the height and width - - def test_boxes_out_of_bounds(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - with pytest.raises(ValueError): - _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_OOB) # Expecting the ValueError due to out of bounds - - def test_transform_with_masks(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_WITH_MASKS) - assert "masks" in annotation - assert annotation["masks"].shape[0] == 1 - - def test_area_calculation_with_masks(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION_WITH_MASKS) - assert annotation["area"] == torch.sum(annotation["masks"]) - - def test_area_calculation_without_masks(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) - area = annotation["boxes"][0, 2] * annotation["boxes"][0, 3] - - assert torch.isclose(annotation["area"], area.unsqueeze(0), atol=1e-5) # Using isclose for floating point comparison - - def test_iscrowd_unchanged(self): - transformations = EXAMPLE_TRANSFORM - at = AlbumentationsTransform(transformations) - _, annotation = at(SAMPLE_IMAGE, SAMPLE_ANNOTATION) - assert "iscrowd" in annotation - assert annotation["iscrowd"] == SAMPLE_ANNOTATION["iscrowd"] From 094cc70bbd67a0630a81cb531607e5cb3dea9d4d Mon Sep 17 00:00:00 2001 From: Christoffer Date: Mon, 9 Oct 2023 16:16:58 +0200 Subject: [PATCH 08/30] added a check for duplicate classes (from polygon and bounding_boxes --- darwin/dataset/utils.py | 6 ++++-- tests/darwin/torch/dataset_test.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 7209c50b2..fb9f057b9 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -180,10 +180,12 @@ def get_classes( else: annotation_types_to_load = [annotation_type] - classes = [] + classes = [] # Use a list to maintain order for atype in annotation_types_to_load: classes_file_path = release_path / f"lists/classes_{atype}.txt" - classes.extend(get_classes_from_file(classes_file_path)) + for cls in get_classes_from_file(classes_file_path): + if cls not in classes: # Only add if it's not already in the list + classes.append(cls) if remove_background and classes and classes[0] == "__background__": classes = classes[1:] diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index 5593dd4ca..99535f4cb 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -77,6 +77,9 @@ def test_should_correctly_create_a_object_detection_dataset( print(root) ds = ObjectDetectionDataset(dataset_path=root, release_name="latest") + print(f"weights : {ds.measure_weights()}") + print(f"Classes : {ds.classes}") + generic_dataset_test(ds, n=20, size=(50, 50)) assert type(ds[0][1]) is dict img, target = ds[0] From 65d43eb84e3def732f0fc17fa836424c9bf913d3 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Mon, 9 Oct 2023 16:30:12 +0200 Subject: [PATCH 09/30] removed code that is not supposed to be in github workflow --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c31739f2f..249d5b8cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,7 +62,6 @@ jobs: pip install wheel pip install --upgrade setuptools pip install --editable ".[test,ml,medical,dev,ocv]" - pip install albumentations pip install pytest - name: Run Tests run: python -m pytest -W ignore::DeprecationWarning From a3dfca99bc2ed5938747ff87cb57f3cb21b8c9f0 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 10 Oct 2023 13:26:56 +0200 Subject: [PATCH 10/30] updated stratified to support bounding_box + polygon --- darwin/dataset/local_dataset.py | 2 ++ darwin/dataset/split_manager.py | 4 ++++ darwin/dataset/utils.py | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index c686f516a..1c84d7d6b 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -93,9 +93,11 @@ def __init__( self.num_classes = len(self.classes) stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) + # Find all the annotations and their corresponding images for stem in stems: + print(f"stems - {stem}") annotation_path = annotations_dir / f"{stem}.json" images = [] for ext in SUPPORTED_IMAGE_EXTENSIONS: diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index b831d2dc0..ad9c2ae05 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -146,6 +146,10 @@ def split_dataset( train_size: int = dataset_size - val_size - test_size split_id = f"{train_size}_{val_size}_{test_size}" + assert train_size > 0, f"Found {train_size} train examples, we need at least 1" + assert val_size > 0, f"Found {val_size} train examples, we need at least 1" + assert test_size > 0, f"Found {test_size} train examples, we need at least 1" + # Compute split id, a combination of val precentage, test percentage and split seed # The split id is used to create a folder with the same name in the "lists" folder if split_seed != 0: diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index fb9f057b9..6ba3d0d4f 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -83,6 +83,9 @@ def extract_classes(annotations_path: Path, annotation_type: str) -> Tuple[Dict[ """ assert annotation_type in ["bounding_box", "polygon", "tag"] + supported_annotation_types = [annotation_type] + if annotation_type == "bounding_box": + supported_annotation_types.append("polygon") classes: Dict[str, Set[int]] = defaultdict(set) indices_to_classes: Dict[int, Set[str]] = defaultdict(set) @@ -93,7 +96,7 @@ def extract_classes(annotations_path: Path, annotation_type: str) -> Tuple[Dict[ continue for annotation in annotation_file.annotations: - if annotation.annotation_class.annotation_type != annotation_type: + if annotation.annotation_class.annotation_type not in supported_annotation_types: continue class_name = annotation.annotation_class.name From 99cb21900ee9e3b5b7c5a632001f11022fcbfed3 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Wed, 11 Oct 2023 11:21:29 +0200 Subject: [PATCH 11/30] removed some printing --- darwin/dataset/local_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 1c84d7d6b..0f917403d 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -97,7 +97,6 @@ def __init__( # Find all the annotations and their corresponding images for stem in stems: - print(f"stems - {stem}") annotation_path = annotations_dir / f"{stem}.json" images = [] for ext in SUPPORTED_IMAGE_EXTENSIONS: From 752b54fb6c227c252c3e22f7d861df3a26228352 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Fri, 13 Oct 2023 15:02:21 +0200 Subject: [PATCH 12/30] changes based on owen's feedback --- darwin/dataset/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 6ba3d0d4f..f14680067 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -134,14 +134,14 @@ def make_class_lists(release_path: Path) -> None: f.write("\n".join(classes_names)) -def get_classes_from_file(path): +def get_classes_from_file(path: Path) -> List[str]: """Helper function to read class names from a file.""" if path.exists(): return path.read_text().splitlines() return [] -def available_annotation_types(release_path): +def available_annotation_types(release_path: Path) -> List[str]: """Returns a list of available annotation types based on the existing files.""" files = [p.name for p in release_path.glob("lists/classes_*.txt")] return [f[len("classes_") : -len(".txt")] for f in files] From 199a71dd139f4f83580023b10a917b20b4b9d42d Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 12:32:11 +0200 Subject: [PATCH 13/30] black formatting --- darwin/dataset/local_dataset.py | 18 ++++----------- darwin/dataset/split_manager.py | 15 ++++--------- tests/darwin/torch/dataset_test.py | 36 ++++++++---------------------- 3 files changed, 17 insertions(+), 52 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 0f917403d..286a004b0 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -87,13 +87,10 @@ def __init__( self.original_annotations_path: Optional[List[Path]] = None # Get the list of classes - self.classes = get_classes( - self.dataset_path, release_name, annotation_type=self.annotation_type, remove_background=True - ) + self.classes = get_classes(self.dataset_path, release_name, annotation_type=self.annotation_type, remove_background=True) self.num_classes = len(self.classes) stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) - # Find all the annotations and their corresponding images for stem in stems: @@ -261,9 +258,7 @@ def parse_json(self, index: int) -> Dict[str, Any]: # Filter out unused classes and annotations of a different type if self.classes is not None: - annotations = [ - a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a) - ] + annotations = [a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a)] return { "image_id": index, "image_path": str(self.images_path[index]), @@ -278,9 +273,7 @@ def annotation_type_supported(self, annotation) -> bool: return annotation_type == "tag" elif self.annotation_type == "bounding_box": is_bounding_box = annotation_type == "bounding_box" - is_supported_polygon = ( - annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data - ) + is_supported_polygon = annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data return is_bounding_box or is_supported_polygon elif self.annotation_type == "polygon": return annotation_type in ["polygon", "complex_polygon"] @@ -446,7 +439,4 @@ def build_stems( if split_path.is_file(): return (e.strip("\n\r") for e in split_path.open()) - raise FileNotFoundError( - "could not find a dataset partition. " - "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`" - ) + raise FileNotFoundError("could not find a dataset partition. " "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`") diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index ad9c2ae05..0594d6e99 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -119,9 +119,7 @@ def split_dataset( try: import sklearn # noqa except ImportError: - raise ImportError( - "Darwin requires scikit-learn to split a dataset. Install it using: pip install scikit-learn" - ) from None + raise ImportError("Darwin requires scikit-learn to split a dataset. Install it using: pip install scikit-learn") from None _validate_split(val_percentage, test_percentage) @@ -410,14 +408,11 @@ def _validate_split(val_percentage: float, test_percentage: float) -> None: raise ValueError(f"Invalid test percentage ({test_percentage}). Must be a float x, where 0 < x < 1.") if val_percentage + test_percentage >= 1: raise ValueError( - f"Invalid combination of validation ({val_percentage}) and test ({test_percentage}) percentages. " - f"Their sum must be a value x, where x < 1." + f"Invalid combination of validation ({val_percentage}) and test ({test_percentage}) percentages. " f"Their sum must be a value x, where x < 1." ) -def _build_split( - split_path: Path, stratified_types: List[str], partitions: List[str] = ["train", "val", "test"] -) -> Split: +def _build_split(split_path: Path, stratified_types: List[str], partitions: List[str] = ["train", "val", "test"]) -> Split: split = Split() split.random = {partition: split_path / f"random_{partition}.txt" for partition in partitions} @@ -426,8 +421,6 @@ def _build_split( stratified_dict: Dict[str, Dict[str, Path]] = {} for stratified_type in stratified_types: - stratified_dict[stratified_type] = { - partition: split_path / f"stratified_{stratified_type}_{partition}.txt" for partition in partitions - } + stratified_dict[stratified_type] = {partition: split_path / f"stratified_{stratified_type}_{partition}.txt" for partition in partitions} split.stratified = stratified_dict return split diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index 99535f4cb..d6d12610a 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -28,18 +28,14 @@ def generic_dataset_test(ds, n, size): class TestClassificationDataset: - def test_should_correctly_create_a_single_label_dataset( - self, team_slug: str, team_extracted_dataset_path: Path - ) -> None: + def test_should_correctly_create_a_single_label_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: root = team_extracted_dataset_path / team_slug / "sl" ds = ClassificationDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) assert not ds.is_multi_label - def test_should_correctly_create_a_multi_label_dataset( - self, team_slug: str, team_extracted_dataset_path: Path - ) -> None: + def test_should_correctly_create_a_multi_label_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: root = team_extracted_dataset_path / team_slug / "ml" ds = ClassificationDataset(dataset_path=root, release_name="latest") @@ -48,9 +44,7 @@ def test_should_correctly_create_a_multi_label_dataset( class TestInstanceSegmentationDataset: - def test_should_correctly_create_a_instance_seg_dataset( - self, team_slug: str, team_extracted_dataset_path: Path - ) -> None: + def test_should_correctly_create_a_instance_seg_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: root = team_extracted_dataset_path / team_slug / "coco" ds = InstanceSegmentationDataset(dataset_path=root, release_name="latest") @@ -59,9 +53,7 @@ def test_should_correctly_create_a_instance_seg_dataset( class TestSemanticSegmentationDataset: - def test_should_correctly_create_a_semantic_seg_dataset( - self, team_slug: str, team_extracted_dataset_path: Path - ) -> None: + def test_should_correctly_create_a_semantic_seg_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: root = team_extracted_dataset_path / team_slug / "coco" ds = SemanticSegmentationDataset(dataset_path=root, release_name="latest") @@ -70,16 +62,10 @@ def test_should_correctly_create_a_semantic_seg_dataset( class TestObjectDetectionDataset: - def test_should_correctly_create_a_object_detection_dataset( - self, team_slug: str, team_extracted_dataset_path: Path - ) -> None: + def test_should_correctly_create_a_object_detection_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: root = team_extracted_dataset_path / team_slug / "coco" - print(root) ds = ObjectDetectionDataset(dataset_path=root, release_name="latest") - print(f"weights : {ds.measure_weights()}") - print(f"Classes : {ds.classes}") - generic_dataset_test(ds, n=20, size=(50, 50)) assert type(ds[0][1]) is dict img, target = ds[0] @@ -109,9 +95,7 @@ def test_exits_when_dataset_does_not_exist_locally(self, v1_or_v2_slug: str, loc get_dataset(f"{v1_or_v2_slug}/test", "classification") exception.assert_called_once_with(1) - def test_loads_classification_dataset( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path - ) -> None: + def test_loads_classification_dataset(self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/sl", "classification") assert isinstance(dataset, ClassificationDataset) assert len(dataset) == 20 @@ -120,9 +104,7 @@ def test_loads_classification_dataset( assert image.size() == (3, 50, 50) assert label.item() == 0 - def test_loads_multi_label_classification_dataset( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path - ) -> None: + def test_loads_multi_label_classification_dataset(self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/ml", "classification") assert isinstance(dataset, ClassificationDataset) assert len(dataset) == 20 @@ -145,7 +127,7 @@ def test_loads_object_detection_dataset_from_bounding_box_annotations( label = {k: v.numpy().tolist() for k, v in label.items()} assert label == { - "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping + "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping "area": [612], "labels": [1], "image_id": [0], @@ -164,7 +146,7 @@ def test_loads_object_detection_dataset_from_polygon_annotations( label = {k: v.numpy().tolist() for k, v in label.items()} assert label == { - "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping + "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping "area": [612], "labels": [1], "image_id": [0], From 56788cfc45360a69b7e535c04a2a6f21ce18fcdf Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 12:54:00 +0200 Subject: [PATCH 14/30] reverted classes functionality to old one, but added the ability to load from multiple types, local_dataset has the logic to add bonding_boxes --- darwin/dataset/local_dataset.py | 8 +++++++- darwin/dataset/utils.py | 21 ++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 286a004b0..63decbbc8 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -86,8 +86,14 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None + annotation_types = [self.annotation_type] + + # We fetch bounding_boxes annotations from selected polygons as well + if self.annotation_type == "bounding_boxes": + annotation_types.append("polygon") + # Get the list of classes - self.classes = get_classes(self.dataset_path, release_name, annotation_type=self.annotation_type, remove_background=True) + self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) self.num_classes = len(self.classes) stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index f14680067..1003e6beb 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -150,7 +150,7 @@ def available_annotation_types(release_path: Path) -> List[str]: def get_classes( dataset_path: PathLike, release_name: Optional[str] = None, - annotation_type: str = "polygon", + annotation_type: Union[str, List[str]] = "polygon", remove_background: bool = True, ) -> List[str]: """ @@ -175,24 +175,23 @@ def get_classes( assert dataset_path is not None dataset_path = Path(dataset_path) release_path = get_release_path(dataset_path, release_name) - - # If annotation_type is a string and is 'bounding_box', also consider polygons if isinstance(annotation_type, str): - if annotation_type == "bounding_box": - annotation_types_to_load = [annotation_type, "polygon"] - else: - annotation_types_to_load = [annotation_type] + annotation_types_to_load = [annotation_type] + else: + annotation_types_to_load = annotation_type classes = [] # Use a list to maintain order for atype in annotation_types_to_load: classes_file_path = release_path / f"lists/classes_{atype}.txt" - for cls in get_classes_from_file(classes_file_path): + + class_per_annotations = get_classes_from_file(classes_file_path) + if remove_background and class_per_annotations and class_per_annotations[0] == "__background__": + class_per_annotations = class_per_annotations[1:] + + for cls in class_per_annotations: if cls not in classes: # Only add if it's not already in the list classes.append(cls) - if remove_background and classes and classes[0] == "__background__": - classes = classes[1:] - available_types = available_annotation_types(release_path) assert len(classes) > 0, f"No classes found for {annotation_type}. Supported types are: {', '.join(available_types)}" From 38552796043520a8aa502c9a9fa81075c017ce19 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 12:54:44 +0200 Subject: [PATCH 15/30] linter check --- darwin/dataset/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 1003e6beb..546eaef93 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -5,7 +5,6 @@ from typing import Any, Dict, Generator, Iterator, List, Optional, Set, Tuple, Union import numpy as np -import orjson as json from PIL import Image as PILImage from rich.live import Live from rich.progress import ProgressBar, track @@ -349,7 +348,7 @@ def get_coco_format_record( for point in path: px.append(point["x"]) py.append(point["y"]) - poly = [(x, y) for x, y in zip(px, py)] + poly = list(zip(px, py)) segmentation.append(list(itertools.chain.from_iterable(poly))) all_px.extend(px) all_py.extend(py) From ba78fe1412b244eca5b3c3c01b8ff3af45fe494c Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 12:57:30 +0200 Subject: [PATCH 16/30] poetry lock fix --- poetry.lock | 141 +--------------------------------------------------- 1 file changed, 1 insertion(+), 140 deletions(-) diff --git a/poetry.lock b/poetry.lock index 911516282..82ed986a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,4 @@ # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. -<<<<<<< HEAD -======= [[package]] name = "albumentations" @@ -25,7 +23,6 @@ scipy = ">=1.1.0" develop = ["imgaug (>=0.4.0)", "pytest"] imgaug = ["imgaug (>=0.4.0)"] tests = ["pytest"] ->>>>>>> origin/master [[package]] name = "argcomplete" @@ -345,42 +342,6 @@ files = [ test = ["pytest (>=6)"] [[package]] -<<<<<<< HEAD -name = "flake8" -version = "6.1.0" -description = "the modular source code checker: pep8 pyflakes and co" -optional = true -python-versions = ">=3.8.1" -files = [ - {file = "flake8-6.1.0-py2.py3-none-any.whl", hash = "sha256:ffdfce58ea94c6580c77888a86506937f9a1a227dfcd15f245d694ae20a6b6e5"}, - {file = "flake8-6.1.0.tar.gz", hash = "sha256:d5b3857f07c030bdb5bf41c7f53799571d75c4491748a3adcd47de929e34cd23"}, -] - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" -pyflakes = ">=3.1.0,<3.2.0" - -[[package]] -name = "flake8-pyproject" -version = "1.2.3" -description = "Flake8 plug-in loading the configuration from pyproject.toml" -optional = true -python-versions = ">= 3.6" -files = [ - {file = "flake8_pyproject-1.2.3-py3-none-any.whl", hash = "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"}, -] - -[package.dependencies] -Flake8 = ">=5" -TOMLi = {version = "*", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["pyTest", "pyTest-cov"] - -[[package]] -======= ->>>>>>> origin/master name = "humanize" version = "4.6.0" description = "Python humanize utilities" @@ -580,20 +541,6 @@ rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx- testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] -<<<<<<< HEAD -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = true -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - -[[package]] -======= ->>>>>>> origin/master name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" @@ -1076,20 +1023,6 @@ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] [[package]] -<<<<<<< HEAD -name = "pycodestyle" -version = "2.11.0" -description = "Python style guide checker" -optional = true -python-versions = ">=3.8" -files = [ - {file = "pycodestyle-2.11.0-py2.py3-none-any.whl", hash = "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8"}, - {file = "pycodestyle-2.11.0.tar.gz", hash = "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0"}, -] - -[[package]] -======= ->>>>>>> origin/master name = "pydantic" version = "1.10.13" description = "Data validation and settings management using python type hints" @@ -1142,20 +1075,6 @@ dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] [[package]] -<<<<<<< HEAD -name = "pyflakes" -version = "3.1.0" -description = "passive checker of Python programs" -optional = true -python-versions = ">=3.8" -files = [ - {file = "pyflakes-3.1.0-py2.py3-none-any.whl", hash = "sha256:4132f6d49cb4dae6819e5379898f2b8cce3c5f23994194c24b77d5da2e36f774"}, - {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, -] - -[[package]] -======= ->>>>>>> origin/master name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." @@ -1438,22 +1357,13 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -<<<<<<< HEAD -version = "13.5.3" -======= version = "13.6.0" ->>>>>>> origin/master description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ -<<<<<<< HEAD - {file = "rich-13.5.3-py3-none-any.whl", hash = "sha256:9257b468badc3d347e146a4faa268ff229039d4c2d176ab0cffb4c4fbc73d5d9"}, - {file = "rich-13.5.3.tar.gz", hash = "sha256:87b43e0543149efa1253f485cd845bb7ee54df16c9617b8a893650ab84b4acb6"}, -======= {file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245"}, {file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef"}, ->>>>>>> origin/master ] [package.dependencies] @@ -1465,14 +1375,6 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] -<<<<<<< HEAD -name = "scikit-learn" -version = "1.3.1" -description = "A set of python modules for machine learning and data mining" -optional = true -python-versions = ">=3.8" -files = [ -======= name = "scikit-image" version = "0.21.0" description = "Image processing in Python" @@ -1529,7 +1431,6 @@ description = "A set of python modules for machine learning and data mining" optional = true python-versions = ">=3.8" files = [ ->>>>>>> origin/master {file = "scikit-learn-1.3.1.tar.gz", hash = "sha256:1a231cced3ee3fa04756b4a7ab532dc9417acd581a330adff5f2c01ac2831fcf"}, {file = "scikit_learn-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3153612ff8d36fa4e35ef8b897167119213698ea78f3fd130b4068e6f8d2da5a"}, {file = "scikit_learn-1.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:6bb9490fdb8e7e00f1354621689187bef3cab289c9b869688f805bf724434755"}, @@ -1830,22 +1731,13 @@ files = [ [[package]] name = "types-requests" -<<<<<<< HEAD -version = "2.31.0.6" -======= version = "2.31.0.8" ->>>>>>> origin/master description = "Typing stubs for requests" optional = false python-versions = ">=3.7" files = [ -<<<<<<< HEAD - {file = "types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0"}, - {file = "types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9"}, -======= {file = "types-requests-2.31.0.8.tar.gz", hash = "sha256:e1b325c687b3494a2f528ab06e411d7092cc546cc9245c000bacc2fca5ae96d4"}, {file = "types_requests-2.31.0.8-py3-none-any.whl", hash = "sha256:39894cbca3fb3d032ed8bdd02275b4273471aa5668564617cc1734b0a65ffdf8"}, ->>>>>>> origin/master ] [package.dependencies] @@ -1863,20 +1755,6 @@ files = [ ] [[package]] -<<<<<<< HEAD -name = "types-urllib3" -version = "1.26.25.14" -description = "Typing stubs for urllib3" -optional = false -python-versions = "*" -files = [ - {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, - {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, -] - -[[package]] -======= ->>>>>>> origin/master name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" @@ -1910,22 +1788,13 @@ numpy = "*" [[package]] name = "urllib3" -<<<<<<< HEAD -version = "2.0.5" -======= version = "2.0.6" ->>>>>>> origin/master description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ -<<<<<<< HEAD - {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, - {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, -======= {file = "urllib3-2.0.6-py3-none-any.whl", hash = "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2"}, {file = "urllib3-2.0.6.tar.gz", hash = "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"}, ->>>>>>> origin/master ] [package.extras] @@ -1966,19 +1835,11 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [extras] dev = ["black", "debugpy", "isort", "mypy", "pytest", "pytest-rerunfailures", "responses"] medical = ["connected-components-3d", "nibabel"] -<<<<<<< HEAD -ml = ["scikit-learn", "torch", "torchvision", "albumentations"] -======= ml = ["albumentations", "scikit-learn", "torch", "torchvision"] ->>>>>>> origin/master ocv = ["opencv-python-headless"] test = ["pytest", "responses"] [metadata] lock-version = "2.0" python-versions = ">=3.7.0,<3.11" -<<<<<<< HEAD -content-hash = "791455d63b5366c2242672436020cfb6ce923be62326751b60ac91deea3587a9" -======= -content-hash = "1158b8ae2bbf51d84bc42543f93bc9bb6013df9dba67dec37a4963e477a143e9" ->>>>>>> origin/master +content-hash = "1158b8ae2bbf51d84bc42543f93bc9bb6013df9dba67dec37a4963e477a143e9" \ No newline at end of file From 081f249bfb806f44a1de2e55a72d37f488750bd2 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 13:02:25 +0200 Subject: [PATCH 17/30] manually fixed some ruff issues --- darwin/dataset/local_dataset.py | 1 - darwin/dataset/split_manager.py | 4 ++-- tests/darwin/torch/dataset_test.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 63decbbc8..33111e684 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -3,7 +3,6 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple import numpy as np -import orjson as json from PIL import Image as PILImage from darwin.dataset.utils import get_classes, get_release_path, load_pil_image diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index 0594d6e99..fbd84b323 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -294,8 +294,8 @@ def _stratify_samples( # Extract entries whose support set is 1 (it would make sklearn crash) and append the to train later unique_labels, count = np.unique(labels, return_counts=True) single_files = [] - for l in unique_labels[count == 1]: - index = np.where(labels == l)[0][0] + for label in unique_labels[count == 1]: + index = np.where(labels == label)[0][0] single_files.append(file_indices[index]) labels = np.delete(labels, index) file_indices = np.delete(file_indices, index) diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index d6d12610a..33fd8053c 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -49,7 +49,7 @@ def test_should_correctly_create_a_instance_seg_dataset(self, team_slug: str, te ds = InstanceSegmentationDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) - assert type(ds[0][1]) is dict + assert isinstance(ds[0][1], dict) class TestSemanticSegmentationDataset: @@ -58,7 +58,7 @@ def test_should_correctly_create_a_semantic_seg_dataset(self, team_slug: str, te ds = SemanticSegmentationDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) - assert type(ds[0][1]) is dict + assert isinstance(ds[0][1], dict) class TestObjectDetectionDataset: @@ -67,7 +67,7 @@ def test_should_correctly_create_a_object_detection_dataset(self, team_slug: str ds = ObjectDetectionDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) - assert type(ds[0][1]) is dict + assert isinstance(ds[0][1], dict) img, target = ds[0] for bbox in target["boxes"]: @@ -251,7 +251,7 @@ def test_loads_semantic_segmentation_dataset_from_polygon_annotations( label = {k: _maybe_tensor_to_list(v) for k, v in label.items()} assert label["image_id"] == [0] - assert type(label["mask"][0]) == list + assert isinstance(label["mask"][0], list) assert label["height"] == 50 assert label["width"] == 50 From c5f72868c14f17e1adc59e26415eac513e9151bc Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 13:22:43 +0200 Subject: [PATCH 18/30] ignoring ruff import * issues in dataset_test.py --- tests/darwin/torch/dataset_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index 33fd8053c..cc8b1dbbe 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -15,7 +15,7 @@ SemanticSegmentationDataset, get_dataset, ) -from tests.fixtures import * +from tests.fixtures import * # noqa: F403 def generic_dataset_test(ds, n, size): From 145ce2000c2311036be719032346ae490bbed649 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 13:32:44 +0200 Subject: [PATCH 19/30] refactored local_dataset class to appease ruff (to long init) --- darwin/dataset/local_dataset.py | 68 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 33111e684..d39bed4e8 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -63,20 +63,6 @@ def __init__( split_type: str = "random", release_name: Optional[str] = None, ): - assert dataset_path is not None - release_path = get_release_path(dataset_path, release_name) - annotations_dir = release_path / "annotations" - assert annotations_dir.exists() - images_dir = dataset_path / "images" - assert images_dir.exists() - - if partition not in ["train", "val", "test", None]: - raise ValueError("partition should be either 'train', 'val', or 'test'") - if split_type not in ["random", "stratified"]: - raise ValueError("split_type should be either 'random', 'stratified'") - if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - self.dataset_path = dataset_path self.annotation_type = annotation_type self.images_path: List[Path] = [] @@ -85,30 +71,42 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None - annotation_types = [self.annotation_type] + release_path, annotations_dir, images_dir = self._initial_setup(dataset_path, release_name) + self._validate_inputs(partition, split_type, annotation_type) + # Get the list of classes + self.num_classes = len(self.classes) + annotation_types = [self.annotation_type] # We fetch bounding_boxes annotations from selected polygons as well if self.annotation_type == "bounding_boxes": annotation_types.append("polygon") - - # Get the list of classes self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) - self.num_classes = len(self.classes) - stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) + self._setup_annotations_and_images(release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type) - # Find all the annotations and their corresponding images + if len(self.images_path) == 0: + raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") + + assert len(self.images_path) == len(self.annotations_path) + + def _validate_inputs(self, partition, split_type, annotation_type): + if partition not in ["train", "val", "test", None]: + raise ValueError("partition should be either 'train', 'val', or 'test'") + if split_type not in ["random", "stratified"]: + raise ValueError("split_type should be either 'random', 'stratified'") + if annotation_type not in ["tag", "polygon", "bounding_box"]: + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + + def _setup_annotations_and_images(self, release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type): + stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) for stem in stems: annotation_path = annotations_dir / f"{stem}.json" - images = [] - for ext in SUPPORTED_IMAGE_EXTENSIONS: - image_path = images_dir / f"{stem}{ext}" - if image_path.exists(): - images.append(image_path) - continue - image_path = images_dir / f"{stem}{ext.upper()}" - if image_path.exists(): - images.append(image_path) + images = [ + image_path + for ext in SUPPORTED_IMAGE_EXTENSIONS + for image_path in [images_dir / f"{stem}{ext}", images_dir / f"{stem}{ext.upper()}"] + if image_path.exists() + ] if len(images) < 1: raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") if len(images) > 1: @@ -116,10 +114,14 @@ def __init__( self.images_path.append(images[0]) self.annotations_path.append(annotation_path) - if len(self.images_path) == 0: - raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") - - assert len(self.images_path) == len(self.annotations_path) + def _initial_setup(self, dataset_path, release_name): + assert dataset_path is not None + release_path = get_release_path(dataset_path, release_name) + annotations_dir = release_path / "annotations" + assert annotations_dir.exists() + images_dir = dataset_path / "images" + assert images_dir.exists() + return release_path, annotations_dir, images_dir def get_img_info(self, index: int) -> Dict[str, Any]: """ From 99c4186c534e0e4c0f165367ff8fb9062987d894 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 14:12:52 +0200 Subject: [PATCH 20/30] added test to extract_classes with multiple annotation types selected --- darwin/dataset/utils.py | 335 ++++++++++++--------- tests/darwin/dataset/dataset_utils_test.py | 83 ++++- 2 files changed, 280 insertions(+), 138 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 546eaef93..e0228bb9d 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -61,48 +61,54 @@ def get_release_path(dataset_path: Path, release_name: Optional[str] = None) -> return release_path -def extract_classes(annotations_path: Path, annotation_type: str) -> Tuple[Dict[str, Set[int]], Dict[int, Set[str]]]: - """ - Given a the GT as json files extracts all classes and an maps images index to classes. - - Parameters - ---------- - annotations_files : Path - Path to the json files with the GT information of each image. - annotation_type : str - Type of annotation to use to extract the Gt information. - - Returns - ------- - Tuple[Dict[str, Set[int]], Dict[int, Set[str]]] - A Tuple where the first element is a ``Dictionary`` where keys are the classes found in the - GT and values are a list of file numbers which contain it; and the second element is - ``Dictionary`` where keys are image indices and values are all classes - contained in that image. - """ - - assert annotation_type in ["bounding_box", "polygon", "tag"] - supported_annotation_types = [annotation_type] - if annotation_type == "bounding_box": - supported_annotation_types.append("polygon") +def extract_classes( + annotations_path: Path, + annotation_type: Union[str, List[str]] + ) -> Tuple[Dict[str, Set[int]], Dict[int, Set[str]]]: + """ + Given the GT as json files extracts all classes and maps images index to classes. + + Parameters + ---------- + annotations_files : Path + Path to the json files with the GT information of each image. + annotation_type : Union[str, List[str]] + Type(s) of annotation to use to extract the GT information. + + Returns + ------- + Tuple[Dict[str, Set[int]], Dict[int, Set[str]]] + A Tuple where the first element is a ``Dictionary`` where keys are the classes found in the + GT and values are a list of file numbers which contain it; and the second element is + ``Dictionary`` where keys are image indices and values are all classes + contained in that image. + """ + + if isinstance(annotation_type, str): + annotation_types_to_load = [annotation_type] + else: + annotation_types_to_load = annotation_type - classes: Dict[str, Set[int]] = defaultdict(set) - indices_to_classes: Dict[int, Set[str]] = defaultdict(set) + for atype in annotation_types_to_load: + assert atype in ["bounding_box", "polygon", "tag"] - for i, file_name in enumerate(sorted(annotations_path.glob("**/*.json"))): - annotation_file = parse_path(file_name) - if not annotation_file: - continue + classes: Dict[str, Set[int]] = defaultdict(set) + indices_to_classes: Dict[int, Set[str]] = defaultdict(set) - for annotation in annotation_file.annotations: - if annotation.annotation_class.annotation_type not in supported_annotation_types: + for i, file_name in enumerate(sorted(annotations_path.glob("**/*.json"))): + annotation_file = parse_path(file_name) + if not annotation_file: continue - class_name = annotation.annotation_class.name - indices_to_classes[i].add(class_name) - classes[class_name].add(i) + for annotation in annotation_file.annotations: + if annotation.annotation_class.annotation_type not in annotation_types_to_load: + continue + + class_name = annotation.annotation_class.name + indices_to_classes[i].add(class_name) + classes[class_name].add(i) - return classes, indices_to_classes + return classes, indices_to_classes def make_class_lists(release_path: Path) -> None: @@ -268,46 +274,15 @@ def get_coco_format_record( image_id: Optional[Union[str, int]] = None, classes: Optional[List[str]] = None, ) -> Dict[str, Any]: - """ - Creates and returns a coco record from the given annotation. - Uses ``BoxMode.XYXY_ABS`` from ``detectron2.structures`` if available, defaults to ``box_mode = 0`` - otherwise. - - Parameters - ---------- - annotation_path : Path - ``Path`` to the annotation file. - annotation_type : str = "polygon" - Type of the annotation we want to retrieve. - image_path : Optional[Path], default: None - ``Path`` to the image the annotation refers to. - image_id : Optional[Union[str, int]], default: None - Id of the image the annotation refers to. - classes : Optional[List[str]], default: None - Classes of the annotation. - - Returns - ------- - Dict[str, Any] - A coco record with the following keys: - - .. code-block:: python - - { - "height": 100, - "width": 100, - "file_name": "a file name", - "image_id": 1, - "annotations": [ ... ] - } - """ assert annotation_type in ["tag", "polygon", "bounding_box"] + try: from detectron2.structures import BoxMode box_mode = BoxMode.XYXY_ABS except ImportError: box_mode = 0 + data = parse_darwin_json(annotation_path) record: Dict[str, Any] = {} @@ -315,6 +290,7 @@ def get_coco_format_record( record["file_name"] = str(image_path) if image_id is not None: record["image_id"] = image_id + record["height"] = data.image_height record["width"] = data.image_width @@ -324,46 +300,65 @@ def get_coco_format_record( if annotation_type not in obj.data: # Allows training object detection with bboxes continue - if classes: - category = classes.index(obj.annotation_class.name) - else: - category = obj.annotation_class.name - new_obj = {"bbox_mode": box_mode, "category_id": category, "iscrowd": 0} - if annotation_type == "polygon": - # Support for complex polygons - if "paths" in obj.data: - paths = obj.data["paths"] - elif "path" in obj.data: - paths = [obj.data["path"]] - else: - raise ValueError("polygon path not found") - all_px, all_py = [], [] - segmentation = [] - - for path in paths: - if len(path) < 3: # Discard polygons with less than 3 points - continue - px, py = [], [] - for point in path: - px.append(point["x"]) - py.append(point["y"]) - poly = list(zip(px, py)) - segmentation.append(list(itertools.chain.from_iterable(poly))) - all_px.extend(px) - all_py.extend(py) - - new_obj["segmentation"] = segmentation - new_obj["bbox"] = [np.min(all_px), np.min(all_py), np.max(all_px), np.max(all_py)] + new_obj = create_polygon_object(obj, box_mode, classes) elif annotation_type == "bounding_box": - bbox = obj.data["bounding_box"] - new_obj["bbox"] = [bbox["x"], bbox["y"], bbox["x"] + bbox["w"], bbox["y"] + bbox["h"]] + new_obj = create_bbox_object(obj, box_mode, classes) + else: + continue objs.append(new_obj) + record["annotations"] = objs return record +def create_polygon_object(obj, box_mode, classes=None): + if "paths" in obj.data: + paths = obj.data["paths"] + elif "path" in obj.data: + paths = [obj.data["path"]] + else: + raise ValueError("polygon path not found") + + all_px, all_py = [], [] + segmentation = [] + + for path in paths: + if len(path) < 3: + continue + px, py = [], [] + for point in path: + px.append(point["x"]) + py.append(point["y"]) + poly = list(zip(px, py)) + segmentation.append(list(itertools.chain.from_iterable(poly))) + all_px.extend(px) + all_py.extend(py) + + new_obj = { + "segmentation": segmentation, + "bbox": [np.min(all_px), np.min(all_py), np.max(all_px), np.max(all_py)], + "bbox_mode": box_mode, + "category_id": classes.index(obj.annotation_class.name) if classes else obj.annotation_class.name, + "iscrowd": 0, + } + + return new_obj + + +def create_bbox_object(obj, box_mode, classes=None): + bbox = obj.data["bounding_box"] + new_obj = { + "bbox": [bbox["x"], bbox["y"], bbox["x"] + bbox["w"], bbox["y"] + bbox["h"]], + "bbox_mode": box_mode, + "category_id": classes.index(obj.annotation_class.name) if classes else obj.annotation_class.name, + "iscrowd": 0, + } + + return new_obj + + def get_annotations( dataset_path: PathLike, partition: Optional[str] = None, @@ -419,12 +414,46 @@ def get_annotations( dataset_path = Path(dataset_path) release_path: Path = get_release_path(dataset_path, release_name) - annotations_dir = release_path / "annotations" assert annotations_dir.exists() images_dir = dataset_path / "images" assert images_dir.exists() + _validate_inputs(partition, split_type, annotation_type) + + classes = get_classes(dataset_path, release_name, annotation_type=annotation_type, remove_background=True) + + if partition: + stems = _get_stems_from_split(release_path, split, split_type, annotation_type, partition) + else: + stems = (e.stem for e in annotations_dir.glob("**/*.json")) + + images_paths, annotations_paths, invalid_annotation_paths = _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_inconsistent_examples) + + print(f"Found {len(invalid_annotation_paths)} invalid annotations") + for p in invalid_annotation_paths: + print(p) + + if len(images_paths) == 0: + raise ValueError(f"Could not find any {SUPPORTED_EXTENSIONS} file" f" in {dataset_path / 'images'}") + + assert len(images_paths) == len(annotations_paths) + + yield from _load_and_format_annotations(images_paths, annotations_paths, annotation_format, annotation_type, classes) + + +def _validate_inputs(partition, split_type, annotation_type): + """ + Validates the input parameters for partition, split_type, and annotation_type. + + Args: + partition (str, None): Dataset partition. Should be 'train', 'val', 'test' or None. + split_type (str, None): Type of dataset split. Can be 'random', 'stratified' or None. + annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. + + Raises: + ValueError: If the input parameters do not match the expected values. + """ if partition not in ["train", "val", "test", None]: raise ValueError("partition should be either 'train', 'val', 'test', or None") if split_type not in ["random", "stratified", None]: @@ -432,37 +461,63 @@ def get_annotations( if annotation_type not in ["tag", "polygon", "bounding_box"]: raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - # Get the list of classes - classes = get_classes(dataset_path, release_name, annotation_type=annotation_type, remove_background=True) - # Get the list of stems - if partition: - # Get the split - if split_type is None: - split_file = f"{partition}.txt" - elif split_type == "random": - split_file = f"{split_type}_{partition}.txt" - elif split_type == "stratified": - split_file = f"{split_type}_{annotation_type}_{partition}.txt" - else: - raise ValueError(f"Invalid split_type ({split_type})") - split_path: Path = release_path / "lists" / str(split) / split_file +def _get_stems_from_split(release_path, split, split_type, annotation_type, partition): + """ + Determines the file stems based on the dataset split and other parameters. + + Args: + release_path (Path): Path to the dataset release. + split (str): Dataset split identifier. + split_type (str, None): Type of dataset split. Can be 'random', 'stratified' or None. + annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. + partition (str, None): Dataset partition. Should be 'train', 'val', 'test' or None. + + Returns: + Generator[str]: File stems for the dataset. + + Raises: + ValueError: If the split_type is invalid. + FileNotFoundError: If the dataset partition file is not found. + """ + if split_type is None: + split_file = f"{partition}.txt" + elif split_type == "random": + split_file = f"{split_type}_{partition}.txt" + elif split_type == "stratified": + split_file = f"{split_type}_{annotation_type}_{partition}.txt" + else: + raise ValueError(f"Invalid split_type ({split_type})") - if split_path.is_file(): - stems: Iterator[str] = (e.rstrip("\n\r") for e in split_path.open()) - else: - raise FileNotFoundError( - "Could not find a dataset partition. ", - "To split the dataset you can use 'split_dataset' from darwin.dataset.split_manager", - ) + split_path: Path = release_path / "lists" / str(split) / split_file + + if split_path.is_file(): + return (e.rstrip("\n\r") for e in split_path.open()) else: - # If the partition is not specified, get all the annotations - stems = (e.stem for e in annotations_dir.glob("**/*.json")) + raise FileNotFoundError( + "Could not find a dataset partition. ", + "To split the dataset you can use 'split_dataset' from darwin.dataset.split_manager", + ) + +def _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_inconsistent_examples): + """ + Maps annotations to their corresponding images based on the file stems. + + Args: + stems (List[str]): List of file stems. + annotations_dir (Path): Directory containing annotation files. + images_dir (Path): Directory containing image files. + ignore_inconsistent_examples (bool): Flag to determine if inconsistent examples should be ignored. + + Returns: + Tuple[List[Path], List[Path], List[Path]]: Lists of paths for images, annotations, and invalid annotations respectively. + + Raises: + ValueError: If there are inconsistencies with the annotations and images. + """ images_paths = [] annotations_paths = [] - - # Find all the annotations and their corresponding images invalid_annotation_paths = [] for stem in stems: annotation_path = annotations_dir / f"{stem}.json" @@ -488,16 +543,26 @@ def get_annotations( images_paths.append(images[0]) annotations_paths.append(annotation_path) - print(f"Found {len(invalid_annotation_paths)} invalid annotations") - for p in invalid_annotation_paths: - print(p) - - if len(images_paths) == 0: - raise ValueError(f"Could not find any {SUPPORTED_EXTENSIONS} file" f" in {dataset_path / 'images'}") + return images_paths, annotations_paths, invalid_annotation_paths - assert len(images_paths) == len(annotations_paths) - # Load and re-format all the annotations +def _load_and_format_annotations(images_paths, annotations_paths, annotation_format, annotation_type, classes): + """ + Loads and formats annotations based on the specified format and type. + + Args: + images_paths (List[Path]): List of paths to image files. + annotations_paths (List[Path]): List of paths to annotation files. + annotation_format (str): Desired output format for annotations. Can be 'coco' or 'darwin'. + annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. + classes (List[str]): List of class names. + + Yields: + Dict: Formatted annotation record. + + Notes: + - If the annotation format is 'coco', video annotations cannot be loaded and will be skipped. + """ if annotation_format == "coco": images_ids = list(range(len(images_paths))) for annotation_path, image_path, image_id in zip(annotations_paths, images_paths, images_ids): diff --git a/tests/darwin/dataset/dataset_utils_test.py b/tests/darwin/dataset/dataset_utils_test.py index 88548bde3..34746d72c 100644 --- a/tests/darwin/dataset/dataset_utils_test.py +++ b/tests/darwin/dataset/dataset_utils_test.py @@ -10,6 +10,7 @@ compute_distributions, exhaust_generator, extract_classes, + get_annotations, get_release_path, sanitize_filename, ) @@ -86,11 +87,11 @@ def test_builds_correct_mapping_dictionaries(self, annotations_path: Path): "image": {"filename": "1.jpg"}, } _create_annotation_file(annotations_path, "1.json", payload) - class_dict, index_dict = extract_classes(annotations_path, "polygon") - assert dict(class_dict) == {"class_1": {0, 1}, "class_3": {0}, "class_5": {1}} - assert dict(index_dict) == {0: {"class_1", "class_3"}, 1: {"class_1", "class_5"}} + assert set(index_dict.keys()) == {0, 1} + assert index_dict[0] == {"class_1", "class_3"} + assert index_dict[1] == {"class_1", "class_5"} class_dict, index_dict = extract_classes(annotations_path, "bounding_box") @@ -102,6 +103,44 @@ def test_builds_correct_mapping_dictionaries(self, annotations_path: Path): assert dict(class_dict) == {"class_4": {0, 1}} assert dict(index_dict) == {0: {"class_4"}, 1: {"class_4"}} + def test_extract_multiple_annotation_types(self, annotations_path: Path): + # Provided payloads + _create_annotation_file(annotations_path, "0.json", { + "annotations": [ + {"name": "class_1", "polygon": {"path": []}}, + {"name": "class_2", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + {"name": "class_3", "polygon": {"path": []}}, + {"name": "class_4", "tag": {}}, + {"name": "class_1", "polygon": {"path": []}}, + ], + "image": {"filename": "0.jpg"}, + }) + _create_annotation_file(annotations_path, "1.json", { + "annotations": [ + {"name": "class_5", "polygon": {"path": []}}, + {"name": "class_6", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + {"name": "class_1", "polygon": {"path": []}}, + {"name": "class_4", "tag": {}}, + {"name": "class_1", "polygon": {"path": []}}, + ], + "image": {"filename": "1.jpg"}, + }) + + # Extracting classes for both bounding_box and polygon annotations + class_dict, index_dict = extract_classes(annotations_path, ["polygon", "bounding_box"]) + + # Assertions + assert set(class_dict.keys()) == {"class_1", "class_2", "class_3", "class_5", "class_6"} + assert class_dict["class_1"] == {0, 1} + assert class_dict["class_2"] == {0} + assert class_dict["class_3"] == {0} + assert class_dict["class_5"] == {1} + assert class_dict["class_6"] == {1} + + assert set(index_dict.keys()) == {0, 1} + assert index_dict[0] == {"class_1", "class_2", "class_3"} + assert index_dict[1] == {"class_1", "class_5", "class_6"} + class TestSanitizeFilename: def test_normal_filenames_stay_untouched(self): @@ -184,3 +223,41 @@ def test_passes_back_exceptions(self): assert len(successes) == 1 assert isinstance(errors[0], Exception) assert errors[0].args[0] == "Test" + +''' +class TestGetAnnotations: + def test_basic_functionality( + self, + team_extracted_dataset_path, + team_dataset_release_path, + annotations_path, + split_path + ): + """ + Basic functionality test for the `get_annotations` function. + """ + + # Test with basic setup + annotations = list(get_annotations(dataset_path=team_extracted_dataset_path)) + assert len(annotations) > 0, "Expected to find some annotations" + + # Add more assertions here to validate the structure of the returned annotations + + def test_partition_handling( + self, + team_extracted_dataset_path, + team_dataset_release_path, + annotations_path, + split_path + ): + """ + Test the partition handling of the `get_annotations` function. + """ + + # Assuming there's a train partition in the test dataset + annotations = list(get_annotations(dataset_path=team_extracted_dataset_path, partition="train")) + assert len(annotations) > 0, "Expected to find some annotations for the train partition" + + # Add more assertions here to validate the structure of the returned annotations + # Repeat for other partitions (e.g., val, test) if present in the mock data +''' \ No newline at end of file From 67dd27497f7cfb1af6b135a419255f4d7097cdd6 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 14:17:52 +0200 Subject: [PATCH 21/30] added stratefied split logic to add polygons to bounding_box stratifed list --- darwin/dataset/local_dataset.py | 4 ++-- darwin/dataset/split_manager.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index d39bed4e8..861c7aed0 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -75,13 +75,13 @@ def __init__( self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes - self.num_classes = len(self.classes) + annotation_types = [self.annotation_type] # We fetch bounding_boxes annotations from selected polygons as well if self.annotation_type == "bounding_boxes": annotation_types.append("polygon") self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) - + self.num_classes = len(self.classes) self._setup_annotations_and_images(release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type) if len(self.images_path) == 0: diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index fbd84b323..5248ebafc 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -230,7 +230,14 @@ def _stratified_split( return for stratified_type in stratified_types: - _, idx_to_classes = extract_classes(annotation_path, stratified_type) + + + if stratified_type == "bounding_box": + class_annotation_types = [stratified_type, "polygon"] + else: + class_annotation_types = stratified_type + + _, idx_to_classes = extract_classes(annotation_path, class_annotation_types) if len(idx_to_classes) == 0: continue From 7e1f194cf9ff1ca5999eec23c97e8c6652927750 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 14:23:55 +0200 Subject: [PATCH 22/30] BLACK --- darwin/dataset/local_dataset.py | 106 +++++++++-- darwin/dataset/split_manager.py | 76 ++++++-- darwin/dataset/utils.py | 210 +++++++++++++-------- tests/darwin/dataset/dataset_utils_test.py | 97 ++++++++-- tests/darwin/torch/dataset_test.py | 89 +++++++-- 5 files changed, 431 insertions(+), 147 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index d3749aa55..ffbe6488c 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -71,7 +71,9 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None - release_path, annotations_dir, images_dir = self._initial_setup(dataset_path, release_name) + release_path, annotations_dir, images_dir = self._initial_setup( + dataset_path, release_name + ) self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes @@ -79,12 +81,28 @@ def __init__( # We fetch bounding_boxes annotations from selected polygons as well if self.annotation_type == "bounding_boxes": annotation_types.append("polygon") - self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) + self.classes = get_classes( + self.dataset_path, + release_name, + annotation_type=annotation_types, + remove_background=True, + ) self.num_classes = len(self.classes) - self._setup_annotations_and_images(release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type) + self._setup_annotations_and_images( + release_path, + annotations_dir, + images_dir, + annotation_type, + split, + partition, + split_type, + ) if len(self.images_path) == 0: - raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") + raise ValueError( + f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", + f" in {images_dir}", + ) assert len(self.images_path) == len(self.annotations_path) @@ -94,22 +112,42 @@ def _validate_inputs(self, partition, split_type, annotation_type): if split_type not in ["random", "stratified"]: raise ValueError("split_type should be either 'random', 'stratified'") if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + raise ValueError( + "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" + ) - def _setup_annotations_and_images(self, release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type): - stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) + def _setup_annotations_and_images( + self, + release_path, + annotations_dir, + images_dir, + annotation_type, + split, + partition, + split_type, + ): + stems = build_stems( + release_path, annotations_dir, annotation_type, split, partition, split_type + ) for stem in stems: annotation_path = annotations_dir / f"{stem}.json" images = [ image_path for ext in SUPPORTED_IMAGE_EXTENSIONS - for image_path in [images_dir / f"{stem}{ext}", images_dir / f"{stem}{ext.upper()}"] + for image_path in [ + images_dir / f"{stem}{ext}", + images_dir / f"{stem}{ext.upper()}", + ] if image_path.exists() ] if len(images) < 1: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") + raise ValueError( + f"Annotation ({annotation_path}) does not have a corresponding image" + ) if len(images) > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") + raise ValueError( + f"Image ({stem}) is present with multiple extensions. This is forbidden." + ) self.images_path.append(images[0]) self.annotations_path.append(annotation_path) @@ -170,7 +208,9 @@ def get_height_and_width(self, index: int) -> Tuple[float, float]: parsed = parse_darwin_json(self.annotations_path[index], index) return parsed.image_height, parsed.image_width - def extend(self, dataset: "LocalDataset", extend_classes: bool = False) -> "LocalDataset": + def extend( + self, dataset: "LocalDataset", extend_classes: bool = False + ) -> "LocalDataset": """ Extends the current dataset with another one. @@ -264,7 +304,12 @@ def parse_json(self, index: int) -> Dict[str, Any]: # Filter out unused classes and annotations of a different type if self.classes is not None: - annotations = [a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a)] + annotations = [ + a + for a in annotations + if a.annotation_class.name in self.classes + and self.annotation_type_supported(a) + ] return { "image_id": index, "image_path": str(self.images_path[index]), @@ -279,14 +324,21 @@ def annotation_type_supported(self, annotation) -> bool: return annotation_type == "tag" elif self.annotation_type == "bounding_box": is_bounding_box = annotation_type == "bounding_box" - is_supported_polygon = annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data + is_supported_polygon = ( + annotation_type in ["polygon", "complex_polygon"] + and "bounding_box" in annotation.data + ) return is_bounding_box or is_supported_polygon elif self.annotation_type == "polygon": return annotation_type in ["polygon", "complex_polygon"] else: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + raise ValueError( + "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" + ) - def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np.ndarray]: + def measure_mean_std( + self, multi_threaded: bool = True + ) -> Tuple[np.ndarray, np.ndarray]: """ Computes mean and std of trained images, given the train loader. @@ -309,7 +361,9 @@ def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np. results = pool.map(self._return_mean, self.images_path) mean = np.sum(np.array(results), axis=0) / len(self.images_path) # Online image_classification deviation - results = pool.starmap(self._return_std, [[item, mean] for item in self.images_path]) + results = pool.starmap( + self._return_std, [[item, mean] for item in self.images_path] + ) std_sum = np.sum(np.array([item[0] for item in results]), axis=0) total_pixel_count = np.sum(np.array([item[1] for item in results])) std = np.sqrt(std_sum / total_pixel_count) @@ -355,14 +409,20 @@ def _compute_weights(labels: List[int]) -> np.ndarray: @staticmethod def _return_mean(image_path: Path) -> np.ndarray: img = np.array(load_pil_image(image_path)) - mean = np.array([np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])]) + mean = np.array( + [np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])] + ) return mean / 255.0 # Loads an image with OpenCV and returns the channel wise std of the image. @staticmethod def _return_std(image_path: Path, mean: np.ndarray) -> Tuple[np.ndarray, float]: img = np.array(load_pil_image(image_path)) / 255.0 - m2 = np.square(np.array([img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]])) + m2 = np.square( + np.array( + [img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]] + ) + ) return np.sum(np.sum(m2, axis=1), 1), m2.size / 3.0 def __getitem__(self, index: int): @@ -432,7 +492,10 @@ def build_stems( """ if partition is None: - return (str(e.relative_to(annotations_dir).parent / e.stem) for e in sorted(annotations_dir.glob("**/*.json"))) + return ( + str(e.relative_to(annotations_dir).parent / e.stem) + for e in sorted(annotations_dir.glob("**/*.json")) + ) if split_type == "random": split_filename = f"{split_type}_{partition}.txt" @@ -445,4 +508,7 @@ def build_stems( if split_path.is_file(): return (e.strip("\n\r") for e in split_path.open()) - raise FileNotFoundError("could not find a dataset partition. " "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`") + raise FileNotFoundError( + "could not find a dataset partition. " + "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`" + ) diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index c9bac9a02..5561631f4 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -119,7 +119,9 @@ def split_dataset( try: import sklearn # noqa except ImportError: - raise ImportError("Darwin requires scikit-learn to split a dataset. Install it using: pip install scikit-learn") from None + raise ImportError( + "Darwin requires scikit-learn to split a dataset. Install it using: pip install scikit-learn" + ) from None _validate_split(val_percentage, test_percentage) @@ -259,13 +261,32 @@ def _stratified_split( else: test_indices.append(idx) - _write_to_file(annotation_path, annotation_files, split[stratified_type]["train"], train_indices) - _write_to_file(annotation_path, annotation_files, split[stratified_type]["val"], val_indices) - _write_to_file(annotation_path, annotation_files, split[stratified_type]["test"], test_indices) + _write_to_file( + annotation_path, + annotation_files, + split[stratified_type]["train"], + train_indices, + ) + _write_to_file( + annotation_path, + annotation_files, + split[stratified_type]["val"], + val_indices, + ) + _write_to_file( + annotation_path, + annotation_files, + split[stratified_type]["test"], + test_indices, + ) def _stratify_samples( - idx_to_classes: Dict[int, Set[str]], split_seed: int, train_size: int, val_size: int, test_size: int + idx_to_classes: Dict[int, Set[str]], + split_seed: int, + train_size: int, + val_size: int, + test_size: int, ) -> Tuple[List[int], List[int], List[int]]: """Splits the list of indices into train, val and test according to their labels (stratified) @@ -337,7 +358,11 @@ def _stratify_samples( # Remove duplicates within the same set # NOTE: doing that earlier (e.g. in _remove_cross_contamination()) would produce mathematical # mistakes in the class balancing between validation and test sets. - return (list(set(X_train.astype(int))), list(set(X_val.astype(int))), list(set(X_test.astype(int)))) + return ( + list(set(X_train.astype(int))), + list(set(X_val.astype(int))), + list(set(X_test.astype(int))), + ) def _remove_cross_contamination( @@ -397,35 +422,58 @@ def _unique(array: np.ndarray) -> np.ndarray: return array[sorted(indexes)] -def _write_to_file(annotation_path: Path, annotation_files: List[Path], file_path: Path, split_idx: Iterable) -> None: +def _write_to_file( + annotation_path: Path, + annotation_files: List[Path], + file_path: Path, + split_idx: Iterable, +) -> None: with open(str(file_path), "w") as f: for i in split_idx: # To deal with recursive search, we want to write the difference between the annotation path # and its parent, without the file extension - stem = str(annotation_files[i]).replace(f"{annotation_path}/", "").rsplit(".json", 1)[0] + stem = ( + str(annotation_files[i]) + .replace(f"{annotation_path}/", "") + .rsplit(".json", 1)[0] + ) f.write(f"{stem}\n") def _validate_split(val_percentage: float, test_percentage: float) -> None: if val_percentage is None or not 0 < val_percentage < 1: - raise ValueError(f"Invalid validation percentage ({val_percentage}). Must be a float x, where 0 < x < 1.") + raise ValueError( + f"Invalid validation percentage ({val_percentage}). Must be a float x, where 0 < x < 1." + ) if test_percentage is None or not 0 < test_percentage < 1: - raise ValueError(f"Invalid test percentage ({test_percentage}). Must be a float x, where 0 < x < 1.") + raise ValueError( + f"Invalid test percentage ({test_percentage}). Must be a float x, where 0 < x < 1." + ) if val_percentage + test_percentage >= 1: raise ValueError( - f"Invalid combination of validation ({val_percentage}) and test ({test_percentage}) percentages. " f"Their sum must be a value x, where x < 1." + f"Invalid combination of validation ({val_percentage}) and test ({test_percentage}) percentages. " + f"Their sum must be a value x, where x < 1." ) -def _build_split(split_path: Path, stratified_types: List[str], partitions: List[str] = ["train", "val", "test"]) -> Split: +def _build_split( + split_path: Path, + stratified_types: List[str], + partitions: List[str] = ["train", "val", "test"], +) -> Split: split = Split() - split.random = {partition: split_path / f"random_{partition}.txt" for partition in partitions} + split.random = { + partition: split_path / f"random_{partition}.txt" for partition in partitions + } if len(stratified_types) == 0: return split stratified_dict: Dict[str, Dict[str, Path]] = {} for stratified_type in stratified_types: - stratified_dict[stratified_type] = {partition: split_path / f"stratified_{stratified_type}_{partition}.txt" for partition in partitions} + stratified_dict[stratified_type] = { + partition: split_path / f"stratified_{stratified_type}_{partition}.txt" + for partition in partitions + } split.stratified = stratified_dict return split diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index e0228bb9d..fc0dfa20c 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -62,53 +62,55 @@ def get_release_path(dataset_path: Path, release_name: Optional[str] = None) -> def extract_classes( - annotations_path: Path, - annotation_type: Union[str, List[str]] - ) -> Tuple[Dict[str, Set[int]], Dict[int, Set[str]]]: - """ - Given the GT as json files extracts all classes and maps images index to classes. - - Parameters - ---------- - annotations_files : Path - Path to the json files with the GT information of each image. - annotation_type : Union[str, List[str]] - Type(s) of annotation to use to extract the GT information. - - Returns - ------- - Tuple[Dict[str, Set[int]], Dict[int, Set[str]]] - A Tuple where the first element is a ``Dictionary`` where keys are the classes found in the - GT and values are a list of file numbers which contain it; and the second element is - ``Dictionary`` where keys are image indices and values are all classes - contained in that image. - """ - - if isinstance(annotation_type, str): - annotation_types_to_load = [annotation_type] - else: - annotation_types_to_load = annotation_type + annotations_path: Path, annotation_type: Union[str, List[str]] +) -> Tuple[Dict[str, Set[int]], Dict[int, Set[str]]]: + """ + Given the GT as json files extracts all classes and maps images index to classes. - for atype in annotation_types_to_load: - assert atype in ["bounding_box", "polygon", "tag"] + Parameters + ---------- + annotations_files : Path + Path to the json files with the GT information of each image. + annotation_type : Union[str, List[str]] + Type(s) of annotation to use to extract the GT information. - classes: Dict[str, Set[int]] = defaultdict(set) - indices_to_classes: Dict[int, Set[str]] = defaultdict(set) + Returns + ------- + Tuple[Dict[str, Set[int]], Dict[int, Set[str]]] + A Tuple where the first element is a ``Dictionary`` where keys are the classes found in the + GT and values are a list of file numbers which contain it; and the second element is + ``Dictionary`` where keys are image indices and values are all classes + contained in that image. + """ - for i, file_name in enumerate(sorted(annotations_path.glob("**/*.json"))): - annotation_file = parse_path(file_name) - if not annotation_file: - continue + if isinstance(annotation_type, str): + annotation_types_to_load = [annotation_type] + else: + annotation_types_to_load = annotation_type - for annotation in annotation_file.annotations: - if annotation.annotation_class.annotation_type not in annotation_types_to_load: - continue + for atype in annotation_types_to_load: + assert atype in ["bounding_box", "polygon", "tag"] + + classes: Dict[str, Set[int]] = defaultdict(set) + indices_to_classes: Dict[int, Set[str]] = defaultdict(set) + + for i, file_name in enumerate(sorted(annotations_path.glob("**/*.json"))): + annotation_file = parse_path(file_name) + if not annotation_file: + continue + + for annotation in annotation_file.annotations: + if ( + annotation.annotation_class.annotation_type + not in annotation_types_to_load + ): + continue - class_name = annotation.annotation_class.name - indices_to_classes[i].add(class_name) - classes[class_name].add(i) + class_name = annotation.annotation_class.name + indices_to_classes[i].add(class_name) + classes[class_name].add(i) - return classes, indices_to_classes + return classes, indices_to_classes def make_class_lists(release_path: Path) -> None: @@ -190,7 +192,11 @@ def get_classes( classes_file_path = release_path / f"lists/classes_{atype}.txt" class_per_annotations = get_classes_from_file(classes_file_path) - if remove_background and class_per_annotations and class_per_annotations[0] == "__background__": + if ( + remove_background + and class_per_annotations + and class_per_annotations[0] == "__background__" + ): class_per_annotations = class_per_annotations[1:] for cls in class_per_annotations: @@ -198,7 +204,9 @@ def get_classes( classes.append(cls) available_types = available_annotation_types(release_path) - assert len(classes) > 0, f"No classes found for {annotation_type}. Supported types are: {', '.join(available_types)}" + assert ( + len(classes) > 0 + ), f"No classes found for {annotation_type}. Supported types are: {', '.join(available_types)}" return classes @@ -211,7 +219,10 @@ def _f(x: Any) -> Any: def exhaust_generator( - progress: Generator, count: int, multi_threaded: bool, worker_count: Optional[int] = None + progress: Generator, + count: int, + multi_threaded: bool, + worker_count: Optional[int] = None, ) -> Tuple[List[Dict[str, Any]], List[Exception]]: """ Exhausts the generator passed as parameter. Can be done multi threaded if desired. @@ -297,7 +308,9 @@ def get_coco_format_record( objs = [] for obj in data.annotations: if annotation_type != obj.annotation_class.annotation_type: - if annotation_type not in obj.data: # Allows training object detection with bboxes + if ( + annotation_type not in obj.data + ): # Allows training object detection with bboxes continue if annotation_type == "polygon": @@ -340,7 +353,9 @@ def create_polygon_object(obj, box_mode, classes=None): "segmentation": segmentation, "bbox": [np.min(all_px), np.min(all_py), np.max(all_px), np.max(all_py)], "bbox_mode": box_mode, - "category_id": classes.index(obj.annotation_class.name) if classes else obj.annotation_class.name, + "category_id": classes.index(obj.annotation_class.name) + if classes + else obj.annotation_class.name, "iscrowd": 0, } @@ -352,7 +367,9 @@ def create_bbox_object(obj, box_mode, classes=None): new_obj = { "bbox": [bbox["x"], bbox["y"], bbox["x"] + bbox["w"], bbox["y"] + bbox["h"]], "bbox_mode": box_mode, - "category_id": classes.index(obj.annotation_class.name) if classes else obj.annotation_class.name, + "category_id": classes.index(obj.annotation_class.name) + if classes + else obj.annotation_class.name, "iscrowd": 0, } @@ -421,36 +438,54 @@ def get_annotations( _validate_inputs(partition, split_type, annotation_type) - classes = get_classes(dataset_path, release_name, annotation_type=annotation_type, remove_background=True) + classes = get_classes( + dataset_path, + release_name, + annotation_type=annotation_type, + remove_background=True, + ) if partition: - stems = _get_stems_from_split(release_path, split, split_type, annotation_type, partition) + stems = _get_stems_from_split( + release_path, split, split_type, annotation_type, partition + ) else: stems = (e.stem for e in annotations_dir.glob("**/*.json")) - images_paths, annotations_paths, invalid_annotation_paths = _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_inconsistent_examples) + ( + images_paths, + annotations_paths, + invalid_annotation_paths, + ) = _map_annotations_to_images( + stems, annotations_dir, images_dir, ignore_inconsistent_examples + ) print(f"Found {len(invalid_annotation_paths)} invalid annotations") for p in invalid_annotation_paths: print(p) if len(images_paths) == 0: - raise ValueError(f"Could not find any {SUPPORTED_EXTENSIONS} file" f" in {dataset_path / 'images'}") + raise ValueError( + f"Could not find any {SUPPORTED_EXTENSIONS} file" + f" in {dataset_path / 'images'}" + ) assert len(images_paths) == len(annotations_paths) - yield from _load_and_format_annotations(images_paths, annotations_paths, annotation_format, annotation_type, classes) + yield from _load_and_format_annotations( + images_paths, annotations_paths, annotation_format, annotation_type, classes + ) def _validate_inputs(partition, split_type, annotation_type): """ Validates the input parameters for partition, split_type, and annotation_type. - + Args: partition (str, None): Dataset partition. Should be 'train', 'val', 'test' or None. split_type (str, None): Type of dataset split. Can be 'random', 'stratified' or None. annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. - + Raises: ValueError: If the input parameters do not match the expected values. """ @@ -459,23 +494,25 @@ def _validate_inputs(partition, split_type, annotation_type): if split_type not in ["random", "stratified", None]: raise ValueError("split_type should be either 'random', 'stratified', or None") if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + raise ValueError( + "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" + ) def _get_stems_from_split(release_path, split, split_type, annotation_type, partition): """ Determines the file stems based on the dataset split and other parameters. - + Args: release_path (Path): Path to the dataset release. split (str): Dataset split identifier. split_type (str, None): Type of dataset split. Can be 'random', 'stratified' or None. annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. partition (str, None): Dataset partition. Should be 'train', 'val', 'test' or None. - + Returns: Generator[str]: File stems for the dataset. - + Raises: ValueError: If the split_type is invalid. FileNotFoundError: If the dataset partition file is not found. @@ -500,19 +537,21 @@ def _get_stems_from_split(release_path, split, split_type, annotation_type, part ) -def _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_inconsistent_examples): +def _map_annotations_to_images( + stems, annotations_dir, images_dir, ignore_inconsistent_examples +): """ Maps annotations to their corresponding images based on the file stems. - + Args: stems (List[str]): List of file stems. annotations_dir (Path): Directory containing annotation files. images_dir (Path): Directory containing image files. ignore_inconsistent_examples (bool): Flag to determine if inconsistent examples should be ignored. - + Returns: Tuple[List[Path], List[Path], List[Path]]: Lists of paths for images, annotations, and invalid annotations respectively. - + Raises: ValueError: If there are inconsistencies with the annotations and images. """ @@ -536,9 +575,13 @@ def _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_incons invalid_annotation_paths.append(annotation_path) continue elif image_count < 1: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") + raise ValueError( + f"Annotation ({annotation_path}) does not have a corresponding image" + ) elif image_count > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") + raise ValueError( + f"Image ({stem}) is present with multiple extensions. This is forbidden." + ) images_paths.append(images[0]) annotations_paths.append(annotation_path) @@ -546,28 +589,34 @@ def _map_annotations_to_images(stems, annotations_dir, images_dir, ignore_incons return images_paths, annotations_paths, invalid_annotation_paths -def _load_and_format_annotations(images_paths, annotations_paths, annotation_format, annotation_type, classes): +def _load_and_format_annotations( + images_paths, annotations_paths, annotation_format, annotation_type, classes +): """ Loads and formats annotations based on the specified format and type. - + Args: images_paths (List[Path]): List of paths to image files. annotations_paths (List[Path]): List of paths to annotation files. annotation_format (str): Desired output format for annotations. Can be 'coco' or 'darwin'. annotation_type (str): Type of annotations. Can be 'tag', 'polygon', or 'bounding_box'. classes (List[str]): List of class names. - + Yields: Dict: Formatted annotation record. - + Notes: - If the annotation format is 'coco', video annotations cannot be loaded and will be skipped. """ if annotation_format == "coco": images_ids = list(range(len(images_paths))) - for annotation_path, image_path, image_id in zip(annotations_paths, images_paths, images_ids): + for annotation_path, image_path, image_id in zip( + annotations_paths, images_paths, images_ids + ): if image_path.suffix.lower() in SUPPORTED_VIDEO_EXTENSIONS: - print(f"[WARNING] Cannot load video annotation into COCO format. Skipping {image_path}") + print( + f"[WARNING] Cannot load video annotation into COCO format. Skipping {image_path}" + ) continue yield get_coco_format_record( annotation_path=annotation_path, @@ -705,24 +754,35 @@ def compute_distributions( - instance_distribution: count of all instances of a given class exist for each partition """ - class_distribution: AnnotationDistribution = {partition: Counter() for partition in partitions} - instance_distribution: AnnotationDistribution = {partition: Counter() for partition in partitions} + class_distribution: AnnotationDistribution = { + partition: Counter() for partition in partitions + } + instance_distribution: AnnotationDistribution = { + partition: Counter() for partition in partitions + } for partition in partitions: for annotation_type in annotation_types: - split_file: Path = split_path / f"stratified_{annotation_type}_{partition}.txt" + split_file: Path = ( + split_path / f"stratified_{annotation_type}_{partition}.txt" + ) if not split_file.exists(): split_file = split_path / f"random_{partition}.txt" stems: List[str] = [e.rstrip("\n\r") for e in split_file.open()] for stem in stems: annotation_path: Path = annotations_dir / f"{stem}.json" - annotation_file: Optional[dt.AnnotationFile] = parse_path(annotation_path) + annotation_file: Optional[dt.AnnotationFile] = parse_path( + annotation_path + ) if annotation_file is None: continue - annotation_class_names: List[str] = [annotation.annotation_class.name for annotation in annotation_file.annotations] + annotation_class_names: List[str] = [ + annotation.annotation_class.name + for annotation in annotation_file.annotations + ] class_distribution[partition] += Counter(set(annotation_class_names)) instance_distribution[partition] += Counter(annotation_class_names) diff --git a/tests/darwin/dataset/dataset_utils_test.py b/tests/darwin/dataset/dataset_utils_test.py index f69e493a7..29157be75 100644 --- a/tests/darwin/dataset/dataset_utils_test.py +++ b/tests/darwin/dataset/dataset_utils_test.py @@ -17,7 +17,9 @@ def open_resource_file(): - resource_file = Path("tests") / "darwin" / "dataset" / "resources" / "stratified_polygon_train" + resource_file = ( + Path("tests") / "darwin" / "dataset" / "resources" / "stratified_polygon_train" + ) return resource_file.open() @@ -31,7 +33,12 @@ def parsed_annotation_file(): {"name": "class_2", "polygon": {"path": []}}, {"name": "class_3", "polygon": {"path": []}}, ], - "image": {"filename": "test.jpg", "height": 1080, "url": "https://darwin.v7labs.com/test.jpg", "width": 1920}, + "image": { + "filename": "test.jpg", + "height": 1080, + "url": "https://darwin.v7labs.com/test.jpg", + "width": 1920, + }, } @@ -66,7 +73,10 @@ def test_builds_correct_mapping_dictionaries(self, annotations_path: Path): payload = { "annotations": [ {"name": "class_1", "polygon": {"path": []}}, - {"name": "class_2", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + { + "name": "class_2", + "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}, + }, {"name": "class_3", "polygon": {"path": []}}, {"name": "class_4", "tag": {}}, {"name": "class_1", "polygon": {"path": []}}, @@ -78,7 +88,10 @@ def test_builds_correct_mapping_dictionaries(self, annotations_path: Path): payload = { "annotations": [ {"name": "class_5", "polygon": {"path": []}}, - {"name": "class_6", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + { + "name": "class_6", + "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}, + }, {"name": "class_1", "polygon": {"path": []}}, {"name": "class_4", "tag": {}}, {"name": "class_1", "polygon": {"path": []}}, @@ -110,7 +123,10 @@ def test_extract_multiple_annotation_types(self, annotations_path: Path): { "annotations": [ {"name": "class_1", "polygon": {"path": []}}, - {"name": "class_2", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + { + "name": "class_2", + "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}, + }, {"name": "class_3", "polygon": {"path": []}}, {"name": "class_4", "tag": {}}, {"name": "class_1", "polygon": {"path": []}}, @@ -124,7 +140,10 @@ def test_extract_multiple_annotation_types(self, annotations_path: Path): { "annotations": [ {"name": "class_5", "polygon": {"path": []}}, - {"name": "class_6", "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}}, + { + "name": "class_6", + "bounding_box": {"x": 0, "y": 0, "w": 100, "h": 100}, + }, {"name": "class_1", "polygon": {"path": []}}, {"name": "class_4", "tag": {}}, {"name": "class_1", "polygon": {"path": []}}, @@ -134,10 +153,18 @@ def test_extract_multiple_annotation_types(self, annotations_path: Path): ) # Extracting classes for both bounding_box and polygon annotations - class_dict, index_dict = extract_classes(annotations_path, ["polygon", "bounding_box"]) + class_dict, index_dict = extract_classes( + annotations_path, ["polygon", "bounding_box"] + ) # Assertions - assert set(class_dict.keys()) == {"class_1", "class_2", "class_3", "class_5", "class_6"} + assert set(class_dict.keys()) == { + "class_1", + "class_2", + "class_3", + "class_5", + "class_6", + } assert class_dict["class_1"] == {0, 1} assert class_dict["class_2"] == {0} assert class_dict["class_3"] == {0} @@ -154,23 +181,53 @@ def test_normal_filenames_stay_untouched(self): assert sanitize_filename("test.jpg") == "test.jpg" def test_special_characters_are_replaced_with_underscores(self): - assert sanitize_filename("2020-06-18T08<50<13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08>50>13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename('2020-06-18T08"50"13.14815Z.json') == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08/50/13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08\\50\\13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08|50|13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08?50?13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" - assert sanitize_filename("2020-06-18T08*50*13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" + assert ( + sanitize_filename("2020-06-18T08<50<13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08>50>13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename('2020-06-18T08"50"13.14815Z.json') + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08/50/13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08\\50\\13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08|50|13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08?50?13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) + assert ( + sanitize_filename("2020-06-18T08*50*13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) @patch("platform.system", return_value="Windows") def test_replace_columns_on_windows(self, mock: MagicMock): - assert sanitize_filename("2020-06-18T08:50:13.14815Z.json") == "2020-06-18T08_50_13.14815Z.json" + assert ( + sanitize_filename("2020-06-18T08:50:13.14815Z.json") + == "2020-06-18T08_50_13.14815Z.json" + ) mock.assert_called_once() @patch("platform.system", return_value="Linux") def test_avoid_replacing_columns_on_non_windows(self, mock: MagicMock): - assert sanitize_filename("2020-06-18T08:50:13.14815Z.json") == "2020-06-18T08:50:13.14815Z.json" + assert ( + sanitize_filename("2020-06-18T08:50:13.14815Z.json") + == "2020-06-18T08:50:13.14815Z.json" + ) mock.assert_called_once() @@ -181,7 +238,9 @@ def _create_annotation_file(annotation_path: Path, filename: str, payload: Dict) class TestGetReleasePath: - def test_defaults_to_latest_version_if_no_version_provided(self, team_dataset_path: Path): + def test_defaults_to_latest_version_if_no_version_provided( + self, team_dataset_path: Path + ): latest_release_path = team_dataset_path / "releases" / "latest" latest_release_path.mkdir(parents=True) assert get_release_path(team_dataset_path) == latest_release_path diff --git a/tests/darwin/torch/dataset_test.py b/tests/darwin/torch/dataset_test.py index cc8b1dbbe..4e1a90088 100644 --- a/tests/darwin/torch/dataset_test.py +++ b/tests/darwin/torch/dataset_test.py @@ -28,14 +28,18 @@ def generic_dataset_test(ds, n, size): class TestClassificationDataset: - def test_should_correctly_create_a_single_label_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: + def test_should_correctly_create_a_single_label_dataset( + self, team_slug: str, team_extracted_dataset_path: Path + ) -> None: root = team_extracted_dataset_path / team_slug / "sl" ds = ClassificationDataset(dataset_path=root, release_name="latest") generic_dataset_test(ds, n=20, size=(50, 50)) assert not ds.is_multi_label - def test_should_correctly_create_a_multi_label_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: + def test_should_correctly_create_a_multi_label_dataset( + self, team_slug: str, team_extracted_dataset_path: Path + ) -> None: root = team_extracted_dataset_path / team_slug / "ml" ds = ClassificationDataset(dataset_path=root, release_name="latest") @@ -44,7 +48,9 @@ def test_should_correctly_create_a_multi_label_dataset(self, team_slug: str, tea class TestInstanceSegmentationDataset: - def test_should_correctly_create_a_instance_seg_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: + def test_should_correctly_create_a_instance_seg_dataset( + self, team_slug: str, team_extracted_dataset_path: Path + ) -> None: root = team_extracted_dataset_path / team_slug / "coco" ds = InstanceSegmentationDataset(dataset_path=root, release_name="latest") @@ -53,7 +59,9 @@ def test_should_correctly_create_a_instance_seg_dataset(self, team_slug: str, te class TestSemanticSegmentationDataset: - def test_should_correctly_create_a_semantic_seg_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: + def test_should_correctly_create_a_semantic_seg_dataset( + self, team_slug: str, team_extracted_dataset_path: Path + ) -> None: root = team_extracted_dataset_path / team_slug / "coco" ds = SemanticSegmentationDataset(dataset_path=root, release_name="latest") @@ -62,7 +70,9 @@ def test_should_correctly_create_a_semantic_seg_dataset(self, team_slug: str, te class TestObjectDetectionDataset: - def test_should_correctly_create_a_object_detection_dataset(self, team_slug: str, team_extracted_dataset_path: Path) -> None: + def test_should_correctly_create_a_object_detection_dataset( + self, team_slug: str, team_extracted_dataset_path: Path + ) -> None: root = team_extracted_dataset_path / team_slug / "coco" ds = ObjectDetectionDataset(dataset_path=root, release_name="latest") @@ -85,17 +95,26 @@ def v1_or_v2_slug(request): class TestGetDataset: - def test_exits_when_dataset_not_supported(self, v1_or_v2_slug: str, local_config_file: Config) -> None: + def test_exits_when_dataset_not_supported( + self, v1_or_v2_slug: str, local_config_file: Config + ) -> None: with patch.object(sys, "exit") as exception: get_dataset(f"{v1_or_v2_slug}/test", "unknown") exception.assert_called_once_with(1) - def test_exits_when_dataset_does_not_exist_locally(self, v1_or_v2_slug: str, local_config_file: Config) -> None: + def test_exits_when_dataset_does_not_exist_locally( + self, v1_or_v2_slug: str, local_config_file: Config + ) -> None: with patch.object(sys, "exit") as exception: get_dataset(f"{v1_or_v2_slug}/test", "classification") exception.assert_called_once_with(1) - def test_loads_classification_dataset(self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path) -> None: + def test_loads_classification_dataset( + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, + ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/sl", "classification") assert isinstance(dataset, ClassificationDataset) assert len(dataset) == 20 @@ -104,7 +123,12 @@ def test_loads_classification_dataset(self, v1_or_v2_slug: str, local_config_fil assert image.size() == (3, 50, 50) assert label.item() == 0 - def test_loads_multi_label_classification_dataset(self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path) -> None: + def test_loads_multi_label_classification_dataset( + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, + ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/ml", "classification") assert isinstance(dataset, ClassificationDataset) assert len(dataset) == 20 @@ -115,7 +139,10 @@ def test_loads_multi_label_classification_dataset(self, v1_or_v2_slug: str, loca assert _maybe_tensor_to_list(label) == [1, 0, 1] def test_loads_object_detection_dataset_from_bounding_box_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/bb", "object-detection") assert isinstance(dataset, ObjectDetectionDataset) @@ -127,7 +154,9 @@ def test_loads_object_detection_dataset_from_bounding_box_annotations( label = {k: v.numpy().tolist() for k, v in label.items()} assert label == { - "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping + "boxes": [ + [4, 33, 17, 16] + ], # we need to account for xywh format and clamping "area": [612], "labels": [1], "image_id": [0], @@ -135,7 +164,10 @@ def test_loads_object_detection_dataset_from_bounding_box_annotations( } def test_loads_object_detection_dataset_from_polygon_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/coco", "object-detection") assert isinstance(dataset, ObjectDetectionDataset) @@ -146,7 +178,9 @@ def test_loads_object_detection_dataset_from_polygon_annotations( label = {k: v.numpy().tolist() for k, v in label.items()} assert label == { - "boxes": [[4, 33, 17, 16]], # we need to account for xywh format and clamping + "boxes": [ + [4, 33, 17, 16] + ], # we need to account for xywh format and clamping "area": [612], "labels": [1], "image_id": [0], @@ -154,7 +188,10 @@ def test_loads_object_detection_dataset_from_polygon_annotations( } def test_loads_object_detection_dataset_from_complex_polygon_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/complex_polygons", "object-detection") assert isinstance(dataset, ObjectDetectionDataset) @@ -173,7 +210,10 @@ def test_loads_object_detection_dataset_from_complex_polygon_annotations( } def test_loads_instance_segmentation_dataset_from_bounding_box_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: # You can load an instance segmentation dataset from an export that only has bounding boxes. # But it will ignore all the annotations, so you'll end up with 0 annotations. @@ -196,7 +236,10 @@ def test_loads_instance_segmentation_dataset_from_bounding_box_annotations( assert label["width"] == 50 def test_loads_instance_segmentation_dataset_from_polygon_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/coco", "instance-segmentation") assert isinstance(dataset, InstanceSegmentationDataset) @@ -217,9 +260,14 @@ def test_loads_instance_segmentation_dataset_from_polygon_annotations( assert label["width"] == 50 def test_loads_instance_segmentation_dataset_from_complex_polygon_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: - dataset = get_dataset(f"{v1_or_v2_slug}/complex_polygons", "instance-segmentation") + dataset = get_dataset( + f"{v1_or_v2_slug}/complex_polygons", "instance-segmentation" + ) assert isinstance(dataset, InstanceSegmentationDataset) assert len(dataset) == 1 @@ -238,7 +286,10 @@ def test_loads_instance_segmentation_dataset_from_complex_polygon_annotations( assert label["width"] == 50 def test_loads_semantic_segmentation_dataset_from_polygon_annotations( - self, v1_or_v2_slug: str, local_config_file: Config, team_extracted_dataset_path: Path + self, + v1_or_v2_slug: str, + local_config_file: Config, + team_extracted_dataset_path: Path, ) -> None: dataset = get_dataset(f"{v1_or_v2_slug}/coco", "semantic-segmentation") assert isinstance(dataset, SemanticSegmentationDataset) From 04de9c526e0d46cd872f43459db5bb975e08173f Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 15:25:34 +0200 Subject: [PATCH 23/30] revrting to old init --- darwin/dataset/local_dataset.py | 63 +++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index ffbe6488c..abf47e416 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -63,6 +63,20 @@ def __init__( split_type: str = "random", release_name: Optional[str] = None, ): + assert dataset_path is not None + release_path = get_release_path(dataset_path, release_name) + annotations_dir = release_path / "annotations" + assert annotations_dir.exists() + images_dir = dataset_path / "images" + assert images_dir.exists() + + if partition not in ["train", "val", "test", None]: + raise ValueError("partition should be either 'train', 'val', or 'test'") + if split_type not in ["random", "stratified"]: + raise ValueError("split_type should be either 'random', 'stratified'") + if annotation_type not in ["tag", "polygon", "bounding_box"]: + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + self.dataset_path = dataset_path self.annotation_type = annotation_type self.images_path: List[Path] = [] @@ -71,38 +85,35 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None - release_path, annotations_dir, images_dir = self._initial_setup( - dataset_path, release_name - ) - self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes - - annotation_types = [self.annotation_type] - # We fetch bounding_boxes annotations from selected polygons as well - if self.annotation_type == "bounding_boxes": - annotation_types.append("polygon") self.classes = get_classes( - self.dataset_path, - release_name, - annotation_type=annotation_types, - remove_background=True, + self.dataset_path, release_name, annotation_type=self.annotation_type, remove_background=True ) self.num_classes = len(self.classes) - self._setup_annotations_and_images( - release_path, - annotations_dir, - images_dir, - annotation_type, - split, - partition, - split_type, - ) + + stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) + + # Find all the annotations and their corresponding images + for stem in stems: + annotation_path = annotations_dir / f"{stem}.json" + images = [] + for ext in SUPPORTED_IMAGE_EXTENSIONS: + image_path = images_dir / f"{stem}{ext}" + if image_path.exists(): + images.append(image_path) + continue + image_path = images_dir / f"{stem}{ext.upper()}" + if image_path.exists(): + images.append(image_path) + if len(images) < 1: + raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") + if len(images) > 1: + raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") + self.images_path.append(images[0]) + self.annotations_path.append(annotation_path) if len(self.images_path) == 0: - raise ValueError( - f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", - f" in {images_dir}", - ) + raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") assert len(self.images_path) == len(self.annotations_path) From 57797cad15f9f84c805c3b0cd7cd16a768de78d9 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 15:26:15 +0200 Subject: [PATCH 24/30] revrting to old init --- darwin/dataset/local_dataset.py | 80 +++++++-------------------------- 1 file changed, 16 insertions(+), 64 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index abf47e416..6973c1aee 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -123,42 +123,22 @@ def _validate_inputs(self, partition, split_type, annotation_type): if split_type not in ["random", "stratified"]: raise ValueError("split_type should be either 'random', 'stratified'") if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError( - "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" - ) + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - def _setup_annotations_and_images( - self, - release_path, - annotations_dir, - images_dir, - annotation_type, - split, - partition, - split_type, - ): - stems = build_stems( - release_path, annotations_dir, annotation_type, split, partition, split_type - ) + def _setup_annotations_and_images(self, release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type): + stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) for stem in stems: annotation_path = annotations_dir / f"{stem}.json" images = [ image_path for ext in SUPPORTED_IMAGE_EXTENSIONS - for image_path in [ - images_dir / f"{stem}{ext}", - images_dir / f"{stem}{ext.upper()}", - ] + for image_path in [images_dir / f"{stem}{ext}", images_dir / f"{stem}{ext.upper()}"] if image_path.exists() ] if len(images) < 1: - raise ValueError( - f"Annotation ({annotation_path}) does not have a corresponding image" - ) + raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") if len(images) > 1: - raise ValueError( - f"Image ({stem}) is present with multiple extensions. This is forbidden." - ) + raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") self.images_path.append(images[0]) self.annotations_path.append(annotation_path) @@ -219,9 +199,7 @@ def get_height_and_width(self, index: int) -> Tuple[float, float]: parsed = parse_darwin_json(self.annotations_path[index], index) return parsed.image_height, parsed.image_width - def extend( - self, dataset: "LocalDataset", extend_classes: bool = False - ) -> "LocalDataset": + def extend(self, dataset: "LocalDataset", extend_classes: bool = False) -> "LocalDataset": """ Extends the current dataset with another one. @@ -315,12 +293,7 @@ def parse_json(self, index: int) -> Dict[str, Any]: # Filter out unused classes and annotations of a different type if self.classes is not None: - annotations = [ - a - for a in annotations - if a.annotation_class.name in self.classes - and self.annotation_type_supported(a) - ] + annotations = [a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a)] return { "image_id": index, "image_path": str(self.images_path[index]), @@ -335,21 +308,14 @@ def annotation_type_supported(self, annotation) -> bool: return annotation_type == "tag" elif self.annotation_type == "bounding_box": is_bounding_box = annotation_type == "bounding_box" - is_supported_polygon = ( - annotation_type in ["polygon", "complex_polygon"] - and "bounding_box" in annotation.data - ) + is_supported_polygon = annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data return is_bounding_box or is_supported_polygon elif self.annotation_type == "polygon": return annotation_type in ["polygon", "complex_polygon"] else: - raise ValueError( - "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" - ) + raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - def measure_mean_std( - self, multi_threaded: bool = True - ) -> Tuple[np.ndarray, np.ndarray]: + def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np.ndarray]: """ Computes mean and std of trained images, given the train loader. @@ -372,9 +338,7 @@ def measure_mean_std( results = pool.map(self._return_mean, self.images_path) mean = np.sum(np.array(results), axis=0) / len(self.images_path) # Online image_classification deviation - results = pool.starmap( - self._return_std, [[item, mean] for item in self.images_path] - ) + results = pool.starmap(self._return_std, [[item, mean] for item in self.images_path]) std_sum = np.sum(np.array([item[0] for item in results]), axis=0) total_pixel_count = np.sum(np.array([item[1] for item in results])) std = np.sqrt(std_sum / total_pixel_count) @@ -420,20 +384,14 @@ def _compute_weights(labels: List[int]) -> np.ndarray: @staticmethod def _return_mean(image_path: Path) -> np.ndarray: img = np.array(load_pil_image(image_path)) - mean = np.array( - [np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])] - ) + mean = np.array([np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])]) return mean / 255.0 # Loads an image with OpenCV and returns the channel wise std of the image. @staticmethod def _return_std(image_path: Path, mean: np.ndarray) -> Tuple[np.ndarray, float]: img = np.array(load_pil_image(image_path)) / 255.0 - m2 = np.square( - np.array( - [img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]] - ) - ) + m2 = np.square(np.array([img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]])) return np.sum(np.sum(m2, axis=1), 1), m2.size / 3.0 def __getitem__(self, index: int): @@ -503,10 +461,7 @@ def build_stems( """ if partition is None: - return ( - str(e.relative_to(annotations_dir).parent / e.stem) - for e in sorted(annotations_dir.glob("**/*.json")) - ) + return (str(e.relative_to(annotations_dir).parent / e.stem) for e in sorted(annotations_dir.glob("**/*.json"))) if split_type == "random": split_filename = f"{split_type}_{partition}.txt" @@ -519,7 +474,4 @@ def build_stems( if split_path.is_file(): return (e.strip("\n\r") for e in split_path.open()) - raise FileNotFoundError( - "could not find a dataset partition. " - "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`" - ) + raise FileNotFoundError("could not find a dataset partition. " "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`") From a4431f8884654de91577344ed3a6f902e3fbcb73 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 15:34:45 +0200 Subject: [PATCH 25/30] made the refactor more like the original --- darwin/dataset/local_dataset.py | 62 ++++++++++----------------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 6973c1aee..9b7cc31f1 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -63,20 +63,6 @@ def __init__( split_type: str = "random", release_name: Optional[str] = None, ): - assert dataset_path is not None - release_path = get_release_path(dataset_path, release_name) - annotations_dir = release_path / "annotations" - assert annotations_dir.exists() - images_dir = dataset_path / "images" - assert images_dir.exists() - - if partition not in ["train", "val", "test", None]: - raise ValueError("partition should be either 'train', 'val', or 'test'") - if split_type not in ["random", "stratified"]: - raise ValueError("split_type should be either 'random', 'stratified'") - if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") - self.dataset_path = dataset_path self.annotation_type = annotation_type self.images_path: List[Path] = [] @@ -85,32 +71,17 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None + release_path, annotations_dir, images_dir = self._initial_setup(dataset_path, release_name) + self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes - self.classes = get_classes( - self.dataset_path, release_name, annotation_type=self.annotation_type, remove_background=True - ) - self.num_classes = len(self.classes) - - stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) - # Find all the annotations and their corresponding images - for stem in stems: - annotation_path = annotations_dir / f"{stem}.json" - images = [] - for ext in SUPPORTED_IMAGE_EXTENSIONS: - image_path = images_dir / f"{stem}{ext}" - if image_path.exists(): - images.append(image_path) - continue - image_path = images_dir / f"{stem}{ext.upper()}" - if image_path.exists(): - images.append(image_path) - if len(images) < 1: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") - if len(images) > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") - self.images_path.append(images[0]) - self.annotations_path.append(annotation_path) + annotation_types = [self.annotation_type] + # We fetch bounding_boxes annotations from selected polygons as well + if self.annotation_type == "bounding_boxes": + annotation_types.append("polygon") + self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) + self.num_classes = len(self.classes) + self._setup_annotations_and_images(release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type) if len(self.images_path) == 0: raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") @@ -129,12 +100,15 @@ def _setup_annotations_and_images(self, release_path, annotations_dir, images_di stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) for stem in stems: annotation_path = annotations_dir / f"{stem}.json" - images = [ - image_path - for ext in SUPPORTED_IMAGE_EXTENSIONS - for image_path in [images_dir / f"{stem}{ext}", images_dir / f"{stem}{ext.upper()}"] - if image_path.exists() - ] + images = [] + for ext in SUPPORTED_IMAGE_EXTENSIONS: + image_path = images_dir / f"{stem}{ext}" + if image_path.exists(): + images.append(image_path) + continue + image_path = images_dir / f"{stem}{ext.upper()}" + if image_path.exists(): + images.append(image_path) if len(images) < 1: raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") if len(images) > 1: From 0ce35b3e59acc44bb2d2afb1a96dc06ef9ea957c Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 15:35:32 +0200 Subject: [PATCH 26/30] added black --- darwin/dataset/local_dataset.py | 101 ++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 19 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 9b7cc31f1..61a952d74 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -71,7 +71,9 @@ def __init__( self.original_images_path: Optional[List[Path]] = None self.original_annotations_path: Optional[List[Path]] = None - release_path, annotations_dir, images_dir = self._initial_setup(dataset_path, release_name) + release_path, annotations_dir, images_dir = self._initial_setup( + dataset_path, release_name + ) self._validate_inputs(partition, split_type, annotation_type) # Get the list of classes @@ -79,12 +81,28 @@ def __init__( # We fetch bounding_boxes annotations from selected polygons as well if self.annotation_type == "bounding_boxes": annotation_types.append("polygon") - self.classes = get_classes(self.dataset_path, release_name, annotation_type=annotation_types, remove_background=True) + self.classes = get_classes( + self.dataset_path, + release_name, + annotation_type=annotation_types, + remove_background=True, + ) self.num_classes = len(self.classes) - self._setup_annotations_and_images(release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type) + self._setup_annotations_and_images( + release_path, + annotations_dir, + images_dir, + annotation_type, + split, + partition, + split_type, + ) if len(self.images_path) == 0: - raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") + raise ValueError( + f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", + f" in {images_dir}", + ) assert len(self.images_path) == len(self.annotations_path) @@ -94,10 +112,23 @@ def _validate_inputs(self, partition, split_type, annotation_type): if split_type not in ["random", "stratified"]: raise ValueError("split_type should be either 'random', 'stratified'") if annotation_type not in ["tag", "polygon", "bounding_box"]: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + raise ValueError( + "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" + ) - def _setup_annotations_and_images(self, release_path, annotations_dir, images_dir, annotation_type, split, partition, split_type): - stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) + def _setup_annotations_and_images( + self, + release_path, + annotations_dir, + images_dir, + annotation_type, + split, + partition, + split_type, + ): + stems = build_stems( + release_path, annotations_dir, annotation_type, split, partition, split_type + ) for stem in stems: annotation_path = annotations_dir / f"{stem}.json" images = [] @@ -110,9 +141,13 @@ def _setup_annotations_and_images(self, release_path, annotations_dir, images_di if image_path.exists(): images.append(image_path) if len(images) < 1: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") + raise ValueError( + f"Annotation ({annotation_path}) does not have a corresponding image" + ) if len(images) > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") + raise ValueError( + f"Image ({stem}) is present with multiple extensions. This is forbidden." + ) self.images_path.append(images[0]) self.annotations_path.append(annotation_path) @@ -173,7 +208,9 @@ def get_height_and_width(self, index: int) -> Tuple[float, float]: parsed = parse_darwin_json(self.annotations_path[index], index) return parsed.image_height, parsed.image_width - def extend(self, dataset: "LocalDataset", extend_classes: bool = False) -> "LocalDataset": + def extend( + self, dataset: "LocalDataset", extend_classes: bool = False + ) -> "LocalDataset": """ Extends the current dataset with another one. @@ -267,7 +304,12 @@ def parse_json(self, index: int) -> Dict[str, Any]: # Filter out unused classes and annotations of a different type if self.classes is not None: - annotations = [a for a in annotations if a.annotation_class.name in self.classes and self.annotation_type_supported(a)] + annotations = [ + a + for a in annotations + if a.annotation_class.name in self.classes + and self.annotation_type_supported(a) + ] return { "image_id": index, "image_path": str(self.images_path[index]), @@ -282,14 +324,21 @@ def annotation_type_supported(self, annotation) -> bool: return annotation_type == "tag" elif self.annotation_type == "bounding_box": is_bounding_box = annotation_type == "bounding_box" - is_supported_polygon = annotation_type in ["polygon", "complex_polygon"] and "bounding_box" in annotation.data + is_supported_polygon = ( + annotation_type in ["polygon", "complex_polygon"] + and "bounding_box" in annotation.data + ) return is_bounding_box or is_supported_polygon elif self.annotation_type == "polygon": return annotation_type in ["polygon", "complex_polygon"] else: - raise ValueError("annotation_type should be either 'tag', 'bounding_box', or 'polygon'") + raise ValueError( + "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" + ) - def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np.ndarray]: + def measure_mean_std( + self, multi_threaded: bool = True + ) -> Tuple[np.ndarray, np.ndarray]: """ Computes mean and std of trained images, given the train loader. @@ -312,7 +361,9 @@ def measure_mean_std(self, multi_threaded: bool = True) -> Tuple[np.ndarray, np. results = pool.map(self._return_mean, self.images_path) mean = np.sum(np.array(results), axis=0) / len(self.images_path) # Online image_classification deviation - results = pool.starmap(self._return_std, [[item, mean] for item in self.images_path]) + results = pool.starmap( + self._return_std, [[item, mean] for item in self.images_path] + ) std_sum = np.sum(np.array([item[0] for item in results]), axis=0) total_pixel_count = np.sum(np.array([item[1] for item in results])) std = np.sqrt(std_sum / total_pixel_count) @@ -358,14 +409,20 @@ def _compute_weights(labels: List[int]) -> np.ndarray: @staticmethod def _return_mean(image_path: Path) -> np.ndarray: img = np.array(load_pil_image(image_path)) - mean = np.array([np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])]) + mean = np.array( + [np.mean(img[:, :, 0]), np.mean(img[:, :, 1]), np.mean(img[:, :, 2])] + ) return mean / 255.0 # Loads an image with OpenCV and returns the channel wise std of the image. @staticmethod def _return_std(image_path: Path, mean: np.ndarray) -> Tuple[np.ndarray, float]: img = np.array(load_pil_image(image_path)) / 255.0 - m2 = np.square(np.array([img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]])) + m2 = np.square( + np.array( + [img[:, :, 0] - mean[0], img[:, :, 1] - mean[1], img[:, :, 2] - mean[2]] + ) + ) return np.sum(np.sum(m2, axis=1), 1), m2.size / 3.0 def __getitem__(self, index: int): @@ -435,7 +492,10 @@ def build_stems( """ if partition is None: - return (str(e.relative_to(annotations_dir).parent / e.stem) for e in sorted(annotations_dir.glob("**/*.json"))) + return ( + str(e.relative_to(annotations_dir).parent / e.stem) + for e in sorted(annotations_dir.glob("**/*.json")) + ) if split_type == "random": split_filename = f"{split_type}_{partition}.txt" @@ -448,4 +508,7 @@ def build_stems( if split_path.is_file(): return (e.strip("\n\r") for e in split_path.open()) - raise FileNotFoundError("could not find a dataset partition. " "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`") + raise FileNotFoundError( + "could not find a dataset partition. " + "Split the dataset using `split_dataset()` from `darwin.dataset.split_manager`" + ) From f2bee699b47d5e8619ecf0afae6062012cd2455f Mon Sep 17 00:00:00 2001 From: Christoffer Date: Tue, 17 Oct 2023 16:51:02 +0200 Subject: [PATCH 27/30] fixed minor issue --- darwin/dataset/local_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 61a952d74..285d3e8ba 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -79,7 +79,7 @@ def __init__( annotation_types = [self.annotation_type] # We fetch bounding_boxes annotations from selected polygons as well - if self.annotation_type == "bounding_boxes": + if self.annotation_type == "bounding_box": annotation_types.append("polygon") self.classes = get_classes( self.dataset_path, From 6aab1ecec422028f54879ad63f7c1ed162ef4530 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Wed, 18 Oct 2023 11:05:12 +0200 Subject: [PATCH 28/30] removed hard val- and test- set requirements --- darwin/dataset/split_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index 5561631f4..96cf6fc49 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -147,8 +147,8 @@ def split_dataset( split_id = f"{train_size}_{val_size}_{test_size}" assert train_size > 0, f"Found {train_size} train examples, we need at least 1" - assert val_size > 0, f"Found {val_size} train examples, we need at least 1" - assert test_size > 0, f"Found {test_size} train examples, we need at least 1" + assert val_size > 0, f"Found {val_size} validation examples, we need at least 1" + assert test_size > 0, f"Found {test_size} test examples, we need at least 1" # Compute split id, a combination of val precentage, test percentage and split seed # The split id is used to create a folder with the same name in the "lists" folder From 0f799a589c884cebd699fb62b2713758f4031464 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Wed, 18 Oct 2023 11:42:16 +0200 Subject: [PATCH 29/30] is exhaust generator code present now? --- darwin/dataset/utils.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index fc0dfa20c..f9802bea8 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -225,25 +225,36 @@ def exhaust_generator( worker_count: Optional[int] = None, ) -> Tuple[List[Dict[str, Any]], List[Exception]]: """ - Exhausts the generator passed as parameter. Can be done multi threaded if desired. + Exhausts the generator passed as parameter. Can be done multi threaded if desired. + Creates and returns a coco record from the given annotation. + + Uses ``BoxMode.XYXY_ABS`` from ``detectron2.structures`` if available, defaults to ``box_mode = 0`` + otherwise. Parameters ---------- - progress : Generator - Generator to exhaust. - count : int - Size of the generator. - multi_threaded : bool - Flag for multi-threaded enabled operations. - worker_count : Optional[int] - Number of workers to use if multi_threaded=True. By default CPU count is used. - + annotation_path : Path + ``Path`` to the annotation file. + annotation_type : str = "polygon" + Type of the annotation we want to retrieve. + image_path : Optional[Path], default: None + ``Path`` to the image the annotation refers to. + image_id : Optional[Union[str, int]], default: None + Id of the image the annotation refers to. + classes : Optional[List[str]], default: None + Classes of the annotation. Returns ------- - List[Dict[str, Any] - List of responses from the generator execution. - List[Exception] - List of exceptions raised during the execution of the generator. + Dict[str, Any] + A coco record with the following keys: + .. code-block:: python + { + "height": 100, + "width": 100, + "file_name": "a file name", + "image_id": 1, + "annotations": [ ... ] + } """ successes = [] errors = [] From 2273fa242084d748f178796c318a9a62dfde12b6 Mon Sep 17 00:00:00 2001 From: Christoffer Date: Thu, 19 Oct 2023 16:34:35 +0200 Subject: [PATCH 30/30] no longer forcing users to have a training split --- darwin/dataset/split_manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/darwin/dataset/split_manager.py b/darwin/dataset/split_manager.py index 96cf6fc49..df7416c40 100644 --- a/darwin/dataset/split_manager.py +++ b/darwin/dataset/split_manager.py @@ -146,10 +146,6 @@ def split_dataset( train_size: int = dataset_size - val_size - test_size split_id = f"{train_size}_{val_size}_{test_size}" - assert train_size > 0, f"Found {train_size} train examples, we need at least 1" - assert val_size > 0, f"Found {val_size} validation examples, we need at least 1" - assert test_size > 0, f"Found {test_size} test examples, we need at least 1" - # Compute split id, a combination of val precentage, test percentage and split seed # The split id is used to create a folder with the same name in the "lists" folder if split_seed != 0: