From 321a890ba85cb8c0266b239bc0e7ea34bccf7ae0 Mon Sep 17 00:00:00 2001 From: cristian Date: Fri, 15 Jan 2021 12:52:47 +0100 Subject: [PATCH] fix: filtering missing keys accounts for values (#1694) * fix: missing keys indices --- jina/executors/indexers/__init__.py | 2 ++ jina/executors/indexers/keyvalue.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/jina/executors/indexers/__init__.py b/jina/executors/indexers/__init__.py index df9e4525ae8c6..1114473a25f03 100644 --- a/jina/executors/indexers/__init__.py +++ b/jina/executors/indexers/__init__.py @@ -165,6 +165,8 @@ def flush(self): def _filter_nonexistent_keys_values(self, keys: Iterator, values: Iterator, existent_keys: Iterator, check_path: str) -> Tuple[List, List]: keys = list(keys) values = list(values) + if len(keys) != len(values): + raise ValueError(f'Keys of length {len(keys)} did not match values of lenth {len(values)}') indices_to_drop = self._get_indices_to_drop(keys, existent_keys, check_path) keys = [keys[i] for i in range(len(keys)) if i not in indices_to_drop] values = [values[i] for i in range(len(values)) if i not in indices_to_drop] diff --git a/jina/executors/indexers/keyvalue.py b/jina/executors/indexers/keyvalue.py index 3f33a1ca86b73..f42e28cc8991f 100644 --- a/jina/executors/indexers/keyvalue.py +++ b/jina/executors/indexers/keyvalue.py @@ -55,7 +55,7 @@ def __init__(self, *args, **kwargs): self._page_size = mmap.ALLOCATIONGRANULARITY def add(self, keys: Iterator[int], values: Iterator[bytes], *args, **kwargs): - if len(keys) != len(values): + if len(list(keys)) != len(list(values)): raise ValueError(f'Len of keys {len(keys)} did not match len of values {len(values)}') for key, value in zip(keys, values): l = len(value) #: the length