From 9ba68e5843e3b93ec280a81a4c5da80e3a0d650f Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 5 Oct 2023 17:24:17 +0100 Subject: [PATCH 1/6] Changes to LocalDataset() & get_annotations() to account for releases pulled with folders --- darwin/dataset/local_dataset.py | 25 +++++++++--------------- darwin/dataset/utils.py | 34 ++++++++++++--------------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index c686f516a..75d9387e8 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -95,23 +95,16 @@ def __init__( stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) # Find all the annotations and their corresponding images - for stem in stems: - annotation_path = annotations_dir / f"{stem}.json" - images = [] - for ext in SUPPORTED_IMAGE_EXTENSIONS: - image_path = images_dir / f"{stem}{ext}" - if image_path.exists(): - images.append(image_path) - continue - image_path = images_dir / f"{stem}{ext.upper()}" - if image_path.exists(): - images.append(image_path) - if len(images) < 1: + invalid_annotation_paths = [] + for annotation_path in annotations_dir.glob("**/*.json"): + darwin_json = parse_darwin_json(annotation_path) + image_path = images_dir / Path(darwin_json.full_path.lstrip('/\\')) + if image_path.exists(): + self.images_path.append(image_path) + self.annotations_path.append(annotation_path) + continue + else: raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") - if len(images) > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") - self.images_path.append(images[0]) - self.annotations_path.append(annotation_path) if len(self.images_path) == 0: raise ValueError(f"Could not find any {SUPPORTED_IMAGE_EXTENSIONS} file", f" in {images_dir}") diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index 3f3bb865f..cdaae8ce2 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -434,29 +434,19 @@ def get_annotations( # Find all the annotations and their corresponding images invalid_annotation_paths = [] - for stem in stems: - annotation_path = annotations_dir / f"{stem}.json" - images = [] - for ext in SUPPORTED_EXTENSIONS: - image_path = images_dir / f"{stem}{ext}" - if image_path.exists(): - images.append(image_path) - continue - image_path = images_dir / f"{stem}{ext.upper()}" - if image_path.exists(): - images.append(image_path) - - image_count = len(images) - if image_count != 1 and ignore_inconsistent_examples: - invalid_annotation_paths.append(annotation_path) + for annotation_path in annotations_dir.glob("**/*.json"): + darwin_json = parse_darwin_json(annotation_path) + image_path = images_dir / Path(darwin_json.full_path.lstrip('/\\')) + if image_path.exists(): + images_paths.append(image_path) + annotations_paths.append(annotation_path) continue - elif image_count < 1: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") - elif image_count > 1: - raise ValueError(f"Image ({stem}) is present with multiple extensions. This is forbidden.") - - images_paths.append(images[0]) - annotations_paths.append(annotation_path) + else: + if ignore_inconsistent_examples: + invalid_annotation_paths.append(annotation_path) + continue + else: + raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") print(f"Found {len(invalid_annotation_paths)} invalid annotations") for p in invalid_annotation_paths: From a6512665c6e4dc98fc63a1a25d7360098943b53a Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Wed, 11 Oct 2023 14:31:58 +0100 Subject: [PATCH 2/6] Preserve order of annotation files in LocalDataset constructor --- darwin/dataset/local_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 75d9387e8..01d2d1d89 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -95,8 +95,7 @@ def __init__( stems = build_stems(release_path, annotations_dir, annotation_type, split, partition, split_type) # Find all the annotations and their corresponding images - invalid_annotation_paths = [] - for annotation_path in annotations_dir.glob("**/*.json"): + for annotation_path in sorted(annotations_dir.glob("**/*.json")): darwin_json = parse_darwin_json(annotation_path) image_path = images_dir / Path(darwin_json.full_path.lstrip('/\\')) if image_path.exists(): From 809800f98697766b9763cb10a44729e8f9f84bae Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 12 Oct 2023 17:30:06 +0100 Subject: [PATCH 3/6] Fixed GH filtering & restored accidentally changed/deleted files --- .github/workflows/EVENT_pull_request.yml | 14 +-- .github/workflows/EVENT_release.yml | 114 +++++++++++------------ .github/workflows/JOB_e2e.yml | 14 +-- deploy/_filter_files.py | 2 +- deploy/nightly_package_setup.py | 44 +++++++++ deploy/revert_nightly_setup.py | 41 ++++++++ 6 files changed, 150 insertions(+), 79 deletions(-) create mode 100644 deploy/nightly_package_setup.py create mode 100644 deploy/revert_nightly_setup.py diff --git a/.github/workflows/EVENT_pull_request.yml b/.github/workflows/EVENT_pull_request.yml index 4c4c1df46..de477c3e6 100644 --- a/.github/workflows/EVENT_pull_request.yml +++ b/.github/workflows/EVENT_pull_request.yml @@ -31,15 +31,15 @@ jobs: with: files: ${{ needs.get_changed_files.outputs.python_changed_files }} - typecheck: - name: Analyse types in python - needs: get_changed_files - uses: ./.github/workflows/JOB_typecheck.yml - with: - files: ${{ needs.get_changed_files.outputs.python_changed_files }} + # typecheck: + # name: Analyse types in python + # needs: get_changed_files + # uses: ./.github/workflows/JOB_typecheck.yml + # with: + # files: ${{ needs.get_changed_files.outputs.python_changed_files }} run_tests: - needs: [format, lint, typecheck] + needs: [format, lint] name: Run tests uses: ./.github/workflows/JOB_tests.yml diff --git a/.github/workflows/EVENT_release.yml b/.github/workflows/EVENT_release.yml index 75a685289..b35b55c14 100644 --- a/.github/workflows/EVENT_release.yml +++ b/.github/workflows/EVENT_release.yml @@ -47,8 +47,9 @@ jobs: needs: validate_tag uses: ./.github/workflows/JOB_tests.yml - build: - needs: validate_tag + release: + needs: [run_tests] + if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -65,80 +66,77 @@ jobs: uses: abatilo/actions-poetry@v2 with: poetry-version: "1.3.1" - - - name: Install dependencies - run: | - poetry install --no-interaction --no-root --all-extras -vvv - poetry build - - - name: Add build to release - uses: actions/upload-release-asset@v1 - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: ./dist/* - asset_name: ${{ env.release_id }}.tar.gz - asset_content_type: application/gzip - - release: - needs: [run_tests, build] - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - steps: - name: Publish on pypi.org env: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - # poetry publish - echo "Publishing to pypi.org - Emulated" + poetry publish --build test_release: - needs: [run_tests, build] + needs: [run_tests] if: startsWith(github.ref, 'refs/tags/test-') runs-on: ubuntu-latest steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - run: pip install pip --upgrade + - name: Setup Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: "1.3.1" + - name: Check secrets are set + run: | + if [[ -z "${{ secrets.TEST_PYPI_USERNAME }}" || -z "${{ secrets.TEST_PYPI_PASSWORD }}" ]]; then + echo "TEST_PYPI_USERNAME and TEST_PYPI_PASSWORD must be set" + exit 1 + fi - name: Publish on test.pypi.org env: POETRY_HTTP_BASIC_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }} POETRY_HTTP_BASIC_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} run: | - poetry publish - echo "Publishing to test.pypi.org - Emulated" + python ./deploy/nightly_package_setup.py + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config http-basic.test-pypi ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }} + + poetry publish --build --repository test-pypi + python ./deploy/revert_nightly_setup.py # Linear tickets update notify_release: - needs: [release, test_release] - if: always() && contains(needs.*.result, 'success') - runs-on: ubuntu-latest - steps: - - name: Notify Slack - uses: ./.github/workflows/JOB_slack_message.yml - with: - icon: ":rocket:" - at_team: ${{ env.is_scheduled || !env.is_draft }} - secrets: inherit - message: | - :tada: *${{ env.release_tag }}* has been released! - :link: - - https://pypi.org/project/darwin-py - - ${{ github.event.release.html_url }} + needs: [release] + if: success() + uses: ./.github/workflows/JOB_slack_message.yml + secrets: inherit + with: + icon: ":rocket:" + at_team: true + message: | + :tada: *${{ inputs.release_tag || github.event.release.tag_name }}* has been released! + :link: + - https://pypi.org/project/darwin-py + - ${{ github.event.release.html_url }} notify_failed_release: - needs: [release, test_release] - if: always() && contains(needs.*.result, 'failure') - runs-on: ubuntu-latest - steps: - - name: Notify Slack - uses: ./.github/workflows/JOB_slack_message.yml - with: - secrets: inherit - icon: ":warning:" - at_team: true - message: | - :warning: *${{ env.release_tag }}* has failed to be released! - - *An error occurred performing release, and you may need to release manually.* - - :link: - - ${{ github.event.release.html_url }} + needs: [release] + if: failure() + uses: ./.github/workflows/JOB_slack_message.yml + secrets: inherit + with: + icon: ":warning:" + at_team: true + message: | + :warning: *${{ inputs.release_tag || github.event.release.tag_name }}* Release has failed to be released! + + *An error occurred performing release, and you may need to release manually.* + + :link: + - ${{ github.event.release.html_url }} diff --git a/.github/workflows/JOB_e2e.yml b/.github/workflows/JOB_e2e.yml index 41ff46cda..b3e231841 100644 --- a/.github/workflows/JOB_e2e.yml +++ b/.github/workflows/JOB_e2e.yml @@ -58,17 +58,5 @@ jobs: :link: - https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }} - :warning: ${{ github.workflow }} failed + :warning: ${{ github.workflow }} failed. - - name: Send Slack Notification - run: | - PAYLOAD=$(cat < None: if file_extension.startswith("."): file_extension = file_extension[1:] - files_out = [file for file in files_in if file.endswith(f".{file_extension}") if "darwin/future" in file] + files_out = [file for file in files_in if file.endswith(f".{file_extension}") and "darwin/future" in file] print(" ".join(files_out)) diff --git a/deploy/nightly_package_setup.py b/deploy/nightly_package_setup.py new file mode 100644 index 000000000..216251ecd --- /dev/null +++ b/deploy/nightly_package_setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Require a single string argument, the new package name +# Example: python change_package_name.py com.example.newname + +from datetime import datetime +from os import linesep +from pathlib import Path + + +def main() -> None: + epoch_timestring = datetime.now().strftime("%s") + + this_file_path = Path(__file__).parent.resolve() + path_to_pyproject = this_file_path / ".." / "pyproject.toml" + path_to_version = this_file_path / ".." / "version.txt" + + try: + assert path_to_pyproject.exists() + except AssertionError: + print("No pyproject.toml found.") + exit(1) + + lines = path_to_pyproject.read_text().splitlines() + lines_to_write = [] + + for line in lines: + if line.startswith("name ="): + lines_to_write.append('name = "darwin-nightly"\n') + elif line.startswith("version ="): + version = line.split("=")[1].strip() + path_to_version.write_text(version) + lines_to_write.append(f'version = "{epoch_timestring}"\n') + else: + lines_to_write.append(line) + + path_to_pyproject.write_text(linesep.join(lines_to_write)) + + print(f"Set build to a nightly in pyproject.toml - darwin-nightly@{epoch_timestring}") + + +if __name__ == "__main__": + main() diff --git a/deploy/revert_nightly_setup.py b/deploy/revert_nightly_setup.py new file mode 100644 index 000000000..12737b6aa --- /dev/null +++ b/deploy/revert_nightly_setup.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from datetime import datetime +from os import path +from pathlib import Path + + +def main() -> None: + new_package_name = "darwin-py" + + this_file_path = Path(__file__).parent.resolve() + path_to_pyproject = this_file_path / ".." / "pyproject.toml" + path_to_version = this_file_path / ".." / "version.txt" + + try: + assert path_to_pyproject.exists() + assert path_to_version.exists() + except AssertionError: + print("No nightly build in place to revert") + exit(1) + + lines = path_to_pyproject.read_text().splitlines() + new_version = path_to_version.read_text().strip() + + lines_to_write = [] + + for line in lines: + if line.startswith("name ="): + line = f'name = "{new_package_name}"\n' + if line.startswith("version ="): + line = f'version = {new_version}\n' + lines_to_write.append(line) + + path_to_pyproject.write_text("\n".join(lines_to_write)) + + print(f"Changed package name to {new_package_name}@{new_version} in pyproject.toml") + + +if __name__ == "__main__": + main() From 23441b21ad692db2a26cd163eea61ef52ffda137 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Thu, 12 Oct 2023 17:36:55 +0100 Subject: [PATCH 4/6] Change to ensure tests can run without formatting & linting --- .github/workflows/EVENT_pull_request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/EVENT_pull_request.yml b/.github/workflows/EVENT_pull_request.yml index de477c3e6..8e8944268 100644 --- a/.github/workflows/EVENT_pull_request.yml +++ b/.github/workflows/EVENT_pull_request.yml @@ -39,7 +39,6 @@ jobs: # files: ${{ needs.get_changed_files.outputs.python_changed_files }} run_tests: - needs: [format, lint] name: Run tests uses: ./.github/workflows/JOB_tests.yml From 464c336b590909fc283ca37dfa5b476f0e8054ba Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Wed, 18 Oct 2023 15:20:10 +0100 Subject: [PATCH 5/6] Added support for JSON streaming to improve speed when parsing Darwin JSON in some scenarios --- darwin/dataset/local_dataset.py | 11 ++- darwin/dataset/utils.py | 6 +- darwin/utils/utils.py | 41 ++++++++++++ poetry.lock | 115 ++++++++++++++++++++++++++++++-- pyproject.toml | 1 + 5 files changed, 162 insertions(+), 12 deletions(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 01d2d1d89..8ac93df91 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -7,7 +7,12 @@ from PIL import Image as PILImage from darwin.dataset.utils import get_classes, get_release_path, load_pil_image -from darwin.utils import SUPPORTED_IMAGE_EXTENSIONS, parse_darwin_json +from darwin.utils import ( + SUPPORTED_IMAGE_EXTENSIONS, + get_image_path_from_stream, + parse_darwin_json, + stream_darwin_json, +) class LocalDataset: @@ -96,8 +101,8 @@ def __init__( # Find all the annotations and their corresponding images for annotation_path in sorted(annotations_dir.glob("**/*.json")): - darwin_json = parse_darwin_json(annotation_path) - image_path = images_dir / Path(darwin_json.full_path.lstrip('/\\')) + darwin_json = stream_darwin_json(annotation_path) + image_path = get_image_path_from_stream(darwin_json, images_dir) if image_path.exists(): self.images_path.append(image_path) self.annotations_path.append(annotation_path) diff --git a/darwin/dataset/utils.py b/darwin/dataset/utils.py index cdaae8ce2..b5eb98e62 100644 --- a/darwin/dataset/utils.py +++ b/darwin/dataset/utils.py @@ -18,9 +18,11 @@ SUPPORTED_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, attempt_decode, + get_image_path_from_stream, is_unix_like_os, parse_darwin_json, ) +from darwin.utils.utils import stream_darwin_json # E.g.: {"partition" => {"class_name" => 123}} AnnotationDistribution = Dict[str, Counter] @@ -435,8 +437,8 @@ def get_annotations( # Find all the annotations and their corresponding images invalid_annotation_paths = [] for annotation_path in annotations_dir.glob("**/*.json"): - darwin_json = parse_darwin_json(annotation_path) - image_path = images_dir / Path(darwin_json.full_path.lstrip('/\\')) + darwin_json = stream_darwin_json(annotation_path) + image_path = get_image_path_from_stream(darwin_json, images_dir) if image_path.exists(): images_paths.append(image_path) annotations_paths.append(annotation_path) diff --git a/darwin/utils/utils.py b/darwin/utils/utils.py index 575c50843..68f81a53f 100644 --- a/darwin/utils/utils.py +++ b/darwin/utils/utils.py @@ -20,9 +20,11 @@ ) import deprecation +import json_stream import numpy as np import orjson as json import requests +from json_stream.base import PersistentStreamingJSONObject from jsonschema import exceptions, validators from requests import Response, request from rich.progress import ProgressType, track @@ -454,6 +456,45 @@ def parse_darwin_json(path: Path, count: Optional[int] = None) -> Optional[dt.An else: return _parse_darwin_image(path, data, count) +def stream_darwin_json(path: Path) -> PersistentStreamingJSONObject: + """ + Returns a Darwin JSON file as a persistent stream. This allows for parsing large files without + loading them entirely into memory. + + Parameters + ---------- + path : Path + Path to the file to parse. + + Returns + ------- + PersistentStreamingJSONObject + A stream of the JSON file. + """ + + with path.open() as infile: + return json_stream.load(infile, persistent=True) + +def get_image_path_from_stream(darwin_json: PersistentStreamingJSONObject, images_dir: Path) -> Path: + """ + Returns the path to the image file associated with the given darwin json file (V1 or V2). + + Parameters + ---------- + darwin_json : PersistentStreamingJSONObject + A stream of the JSON file. + images_dir : Path + Path to the directory containing the images. + + Returns + ------- + Path + Path to the image file. + """ + try: + return images_dir / (Path(darwin_json['item']['path'].lstrip('/\\'))) / Path(darwin_json['item']['name']) + except KeyError: + return images_dir / (Path(darwin_json['image']['path'].lstrip('/\\'))) / Path(darwin_json['image']['filename']) def _parse_darwin_v2(path: Path, data: Dict[str, Any]) -> dt.AnnotationFile: item = data["item"] diff --git a/poetry.lock b/poetry.lock index 4e27775fc..41a6033b3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "argcomplete" @@ -422,6 +422,92 @@ files = [ {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, ] +[[package]] +name = "json-stream" +version = "2.3.2" +description = "Streaming JSON encoder and decoder" +optional = false +python-versions = "<4,>=3.5" +files = [ + {file = "json-stream-2.3.2.tar.gz", hash = "sha256:b8b450ea8e8e3c239e9e7e38d12fed934e77a353c14b297f8ee345a5ceb25b91"}, + {file = "json_stream-2.3.2-py3-none-any.whl", hash = "sha256:236b8e08e2761b209816452a3527355e757913d833e6802b68a034a13b8bd3ac"}, +] + +[package.dependencies] +json-stream-rs-tokenizer = ">=0.4.17" + +[package.extras] +httpx = ["httpx"] +requests = ["requests"] + +[[package]] +name = "json-stream-rs-tokenizer" +version = "0.4.25" +description = "A faster tokenizer for the json-stream Python library" +optional = false +python-versions = ">=3.7,<4" +files = [ + {file = "json-stream-rs-tokenizer-0.4.25.tar.gz", hash = "sha256:6382f2f71a2ef17cef1cc85165d770a3e09360fe211576824a14089f7cb0e422"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1dc6c02532843750e4a006e84b9418efdf3a503c9dd2dea98b0814fa2e4dbc2b"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:62d275ceee82a68c3630099bfc98785fd7908df7f55db18ec025bc36bef2277d"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a20ad9b23d08ce8679913d27590d49388950bd44f0219447c3bc7e39f3988676"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c33f2c07c3960c28b29d12fc61675ccaffa910008943f7532dec4e29a9f71d3"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9e6961a76bb34ae43d49e170074a6b0dedb6db70f455b34b91db737feaa63e"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d74b79d645173c60958722ed14025cecc9799d9e94f060749095f5fe3716aba7"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-win32.whl", hash = "sha256:b2982d1f5e5c56cb5510815c68737f1574a1b0216b9692a0ec4176cce8529b9e"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp310-cp310-win_amd64.whl", hash = "sha256:f1b460e4941fc1b3c40034b2b859595914103e2689f7195e951a41c4dcd3f7f5"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ab8cadc72d50d51fedaa212b999ea07d6aa6c61dbf819e2743730f4fb1859c88"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5c1929ad3c08d6ffc8f5bdd981256e38a4f3ec7a23372f3169ecb904b7925f81"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef674b7cf85b5918ba8a6d028b2557818450e1d7a3c2c7c48a11e84d549d36cf"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9014c576134437db1e6f525a6503bd180a1d5de08edaca58ee3526f0faccd4e"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd5ae65bf19b02c0e0d0814d78af75ef7e83bacc7e3c575544667f1fabda96cd"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:778f42ee54528a2b7c433d6da4c52ffbccd85ddb27ae0e038e3c024289cea915"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-win32.whl", hash = "sha256:c4ef87dc0cb0f8a755de3d3253f328eef22c156db6abba6d446c7921336d4337"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp311-cp311-win_amd64.whl", hash = "sha256:ca42ee5b7e534938826058249f9330bc10f02b6554a845edfd525da37d2f16e1"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:61d38c1e348a5958f195193e7e6d95858dc009895ef0009f4303e8d9a0223744"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8518ba5e5fba8da438a8e33a5dd8d87ac0651d74158d287030ef9b9e56cb0d"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cee037e98a1f7bdac531b2d26aab77173fd0ae8600c8d06ecd053fa6947cdae8"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f93ad9b24746e44f7c121b93c4abc75933241bb0182e4a8ef4e13fd4a30c7c5d"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:31025ca031a6c8fb970193e049e1146fef2b4c71ac29ee9c503e5992d827d019"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-win32.whl", hash = "sha256:9e0a74a1d9afedcf6dd20c3b3e44ca8162e44fb8ddbaffda74b0b9561e68a08e"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp37-cp37m-win_amd64.whl", hash = "sha256:d1947c8d5aa26a6b73e01e0420b3936a731df516ce7413b3dcaadb1085a9ef57"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:293d8e8cac006fae6bf8676d0c2440c5f3921763420bd2a4deacfd8153289700"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e6b9ef224a5d82416135a082977013f8e3b64269756801fbcbfa7d2303fb5b47"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:395b897b88f1b7d6cd91046f53ed8d61e921cc585c2dbd29711fb0846cca42ca"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93db339a6cd51879b211c87d4918176a6520db577ecfb181bacacd3a4eda59f"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2c0b9521041195aa91bfa66ac8043cc881517ec5a13192add6b90e1c7cb6ac0"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:12a8cc62b158df1f950bad577c0e600e1291a99f5d7e8b38c360185495e53db2"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-win32.whl", hash = "sha256:5d871ef12fcb9d5c2a979cc9b0a210d94619fc06817c352e938bb8bde8a3db1a"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp38-cp38-win_amd64.whl", hash = "sha256:a91c65cc13e4d125e4e85d40092e0473e346e0b4885451e23e68083111a4c6ba"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3fb6d66640af7593ae5873a600ed38e6081bb5286e668228caebe71b9d1a922"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f8820fc0f72e183e642374c6484223b6218dedacd8d99cb4b69755ce75c68614"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf24f2ac79425af04c203620f61f6a9e7f8512051ef24614c1551b7e6401b291"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:673a46d2fde818650f7955d1ed857a784b62c53e2f02a57ecb2566eeb84bc1dc"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b9a412936481417c1a42b681c1664166ccb5b9128093eb05ebb4f2be7ae57ef"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:288a013d10081fb2e917db848d0a161b87dadd7922c5372e3b60e5506cd172a0"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-win32.whl", hash = "sha256:feda842ccce2a10ed90f3ffc087681ab6a731bb80ed3a52dbe0c01e41db02929"}, + {file = "json_stream_rs_tokenizer-0.4.25-cp39-cp39-win_amd64.whl", hash = "sha256:1beb4fddcdf9fdd787d502532ceedeb3ce60a59ab1c8b19e0a3fd63367ab9e3c"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eaef5abd7fdf496bf00a2d9310d60c52d982ea6fd079578f2f8c969a0c65413e"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643f50bb351f4c81488d98b50499ce4d30f767a15a47a1baf80a990e6331836e"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05df3f38088ec79f8ac5b154f73a55b51e2869f0b8b68f82e807779e30f29110"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6e73e84a38252edfb3f80c9bc1247639530f21108786982c5156475da884756"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8f886d294dd13bf07e66808823c38d3d1c55221c4f24b4e5ce6bc30addfa387b"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4f863b65d557b9ce0edbe633cb7a37a6af1670d1e1a3d17638ed3ab8971fb6ae"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2059224a321ac2ef9585130ee1d766d3294054fe7db4554075d3975762a19cb"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fae3b89882013e3c01aa0c579de683d1fc2d81a0ba573f9b196213afba67d763"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac36c8018abefe6b31dee9d2b27edea74747fbe3dd517da618a6ceead89f63fc"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:86e11b136f2e58147f3ec3fb0d0148eb7ebb67c52074766a13583af4ab060e72"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ee2d5b7f31aaf897467765eb8cdd863715654d37af0c3a38d3e9cd4a67161eb1"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8266fbe792c34102525b4b152edd122a03f3501af139f75a670ed187ea793c0"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:907d1f5bed0d721a2ac1eb30d58d4117d55267059e17e055abe2b3e58e7517a4"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7568a7df0514e5fa356055dfdf5a035d118b3b82228104e7a484b4ddf7f94442"}, + {file = "json_stream_rs_tokenizer-0.4.25-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:5be0c4d185d63d1d25c820117b1081336cc1db8dad620266e0973fe2f47768b4"}, +] + +[package.extras] +benchmark = ["contexttimer (>=0.3,<0.4)", "json-stream-to-standard-types (>=0.1,<0.2)", "si-prefix (>=1.2,<2)", "tqdm (>=4.64,<5)", "typer (>=0.6,<0.7)"] +test = ["json-stream (==2.3.2)", "json-stream-rs-tokenizer[benchmark]", "pytest (>7.1,<8)"] + [[package]] name = "jsonschema" version = "4.17.3" @@ -709,12 +795,12 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.21.0", markers = "python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, + {version = ">=1.21.0", markers = "python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\" and python_version >= \"3.7\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""}, - {version = ">=1.17.0", markers = "python_version >= \"3.7\""}, - {version = ">=1.17.3", markers = "python_version >= \"3.8\""}, + {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and python_version < \"3.10\" and platform_machine == \"aarch64\" and python_version >= \"3.7\" or python_version < \"3.10\" and platform_system != \"Darwin\" and python_version >= \"3.9\" or python_version < \"3.10\" and python_version >= \"3.9\" and platform_machine != \"arm64\" or python_version > \"3.9\" and python_version < \"3.10\""}, + {version = ">=1.17.3", markers = "(platform_system != \"Darwin\" and platform_system != \"Linux\") and python_version >= \"3.8\" and python_version < \"3.9\" or platform_system != \"Darwin\" and python_version >= \"3.8\" and python_version < \"3.9\" and platform_machine != \"aarch64\" or platform_machine != \"arm64\" and python_version >= \"3.8\" and python_version < \"3.9\" and platform_system != \"Linux\" or (platform_machine != \"arm64\" and platform_machine != \"aarch64\") and python_version >= \"3.8\" and python_version < \"3.9\""}, + {version = ">=1.17.0", markers = "(platform_system != \"Darwin\" and platform_system != \"Linux\") and python_version >= \"3.7\" and python_version < \"3.8\" or platform_system != \"Darwin\" and python_version >= \"3.7\" and python_version < \"3.8\" and platform_machine != \"aarch64\" or platform_machine != \"arm64\" and python_version >= \"3.7\" and python_version < \"3.8\" and platform_system != \"Linux\" or (platform_machine != \"arm64\" and platform_machine != \"aarch64\") and python_version >= \"3.7\" and python_version < \"3.8\""}, ] [[package]] @@ -1124,6 +1210,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1131,8 +1218,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1149,6 +1243,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1156,6 +1251,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1240,6 +1336,11 @@ files = [ {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f66eddfda9d45dd6cadcd706b65669ce1df84b8549875691b1f403730bdef217"}, {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6448c37741145b241eeac617028ba6ec2119e1339b1385c9720dae31367f2be"}, {file = "scikit_learn-1.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c413c2c850241998168bbb3bd1bb59ff03b1195a53864f0b80ab092071af6028"}, + {file = "scikit_learn-1.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ef540e09873e31569bc8b02c8a9f745ee04d8e1263255a15c9969f6f5caa627f"}, + {file = "scikit_learn-1.3.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9147a3a4df4d401e618713880be023e36109c85d8569b3bf5377e6cd3fecdeac"}, + {file = "scikit_learn-1.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2cd3634695ad192bf71645702b3df498bd1e246fc2d529effdb45a06ab028b4"}, + {file = "scikit_learn-1.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c275a06c5190c5ce00af0acbb61c06374087949f643ef32d355ece12c4db043"}, + {file = "scikit_learn-1.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:0e1aa8f206d0de814b81b41d60c1ce31f7f2c7354597af38fae46d9c47c45122"}, {file = "scikit_learn-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:52b77cc08bd555969ec5150788ed50276f5ef83abb72e6f469c5b91a0009bbca"}, {file = "scikit_learn-1.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a683394bc3f80b7c312c27f9b14ebea7766b1f0a34faf1a2e9158d80e860ec26"}, {file = "scikit_learn-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15d964d9eb181c79c190d3dbc2fff7338786bf017e9039571418a1d53dab236"}, @@ -1623,4 +1724,4 @@ test = ["pytest", "responses"] [metadata] lock-version = "2.0" python-versions = ">=3.7.0,<3.11" -content-hash = "9865d21e7a68ded971e8b79d8d7c3b3ec669bb97eadce284fda60b892540ad2d" +content-hash = "1b01a0de65a14aaa6261a525c3399fb3055eff7e02a5afdc1df1168fb46a1ece" diff --git a/pyproject.toml b/pyproject.toml index d0d834d87..39f75ab1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,7 @@ python-dotenv = { version = "^1.0.0", python = ">3.8" } opencv-python-headless = { version = "^4.8.0.76", optional = true } pyyaml = "^6.0.1" pytest-rerunfailures = { version = "^12.0", optional = true } +json-stream = "^2.3.2" [tool.poetry.extras] dev = [ From 7171b3cb7eb3c355c671169c6f25ff14b67b6143 Mon Sep 17 00:00:00 2001 From: John Wilkie Date: Mon, 23 Oct 2023 12:32:38 +0100 Subject: [PATCH 6/6] Updated changes to be compatible with AI-1260 --- darwin/dataset/local_dataset.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/darwin/dataset/local_dataset.py b/darwin/dataset/local_dataset.py index 795a26213..d79ecae76 100644 --- a/darwin/dataset/local_dataset.py +++ b/darwin/dataset/local_dataset.py @@ -121,6 +121,16 @@ def _validate_inputs(self, partition, split_type, annotation_type): "annotation_type should be either 'tag', 'bounding_box', or 'polygon'" ) + def _setup_annotations_and_images( + self, + release_path, + annotations_dir, + images_dir, + annotation_type, + split, + partition, + split_type, + ): # Find all the annotations and their corresponding images for annotation_path in sorted(annotations_dir.glob("**/*.json")): darwin_json = stream_darwin_json(annotation_path) @@ -130,7 +140,9 @@ def _validate_inputs(self, partition, split_type, annotation_type): self.annotations_path.append(annotation_path) continue else: - raise ValueError(f"Annotation ({annotation_path}) does not have a corresponding image") + raise ValueError( + f"Annotation ({annotation_path}) does not have a corresponding image" + ) def _initial_setup(self, dataset_path, release_name): assert dataset_path is not None