Merge pull request #2855 from activeloopai/orig_dataset #1565
GitHub Actions / JUnit Test Report
failed
May 15, 2024 in 0s
28008 tests run, 16601 passed, 11399 skipped, 8 failed.
Annotations
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.6, ...} == {'recall@1': ...@3': 0.6, ...}
Omitting 4 identical items, use -vv to show
Differing items:
{'recall@100': 0.8} != {'recall@100': 0.9}
{'recall@1': 0.2} != {'recall@1': 0.4}
Full diff:
{
- 'recall@1': 0.4,
? ^
+ 'recall@1': 0.2,
? ^
'recall@10': 0.6,
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
'recall@3': 0.6,
'recall@5': 0.6,
'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmpb25a_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmpb25a_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.6, ...} == {'recall@1': ...@3': 0.6, ...}
E Omitting 4 identical items, use -vv to show
E Differing items:
E {'recall@100': 0.8} != {'recall@100': 0.9}
E {'recall@1': 0.2} != {'recall@1': 0.4}
E Full diff:
E {
E - 'recall@1': 0.4,
E ? ^
E + 'recall@1': 0.2,
E ? ^
E 'recall@10': 0.6,
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E 'recall@3': 0.6,
E 'recall@5': 0.6,
E 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
Differing items:
{'recall@10': 0.5} != {'recall@10': 0.6}
{'recall@5': 0.5} != {'recall@5': 0.6}
{'recall@3': 0.5} != {'recall@3': 0.6}
{'recall@1': 0.2} != {'recall@1': 0.4}
{'recall@100': 0.8} != {'recall@100': 0.9}
{'recall@50': 0.6} != {'recall@50': 0.7}
Full diff:
{
- 'recall@1': 0.4,
? ^
+ 'recall@1': 0.2,
? ^
- 'recall@10': 0.6,
? ^
+ 'recall@10': 0.5,
? ^
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.5,
? ^
+ 'recall@5': 0.5,
- 'recall@5': 0.6,
+ 'recall@50': 0.6,
? +
- 'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmp4db7_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmp4db7_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
E Differing items:
E {'recall@10': 0.5} != {'recall@10': 0.6}
E {'recall@5': 0.5} != {'recall@5': 0.6}
E {'recall@3': 0.5} != {'recall@3': 0.6}
E {'recall@1': 0.2} != {'recall@1': 0.4}
E {'recall@100': 0.8} != {'recall@100': 0.9}
E {'recall@50': 0.6} != {'recall@50': 0.7}
E Full diff:
E {
E - 'recall@1': 0.4,
E ? ^
E + 'recall@1': 0.2,
E ? ^
E - 'recall@10': 0.6,
E ? ^
E + 'recall@10': 0.5,
E ? ^
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.5,
E ? ^
E + 'recall@5': 0.5,
E - 'recall@5': 0.6,
E + 'recall@50': 0.6,
E ? +
E - 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
Differing items:
{'recall@10': 0.5} != {'recall@10': 0.6}
{'recall@100': 0.8} != {'recall@100': 0.9}
{'recall@5': 0.5} != {'recall@5': 0.6}
{'recall@1': 0.2} != {'recall@1': 0.4}
{'recall@50': 0.6} != {'recall@50': 0.7}
{'recall@3': 0.5} != {'recall@3': 0.6}
Full diff:
{
- 'recall@1': 0.4,
? ^
+ 'recall@1': 0.2,
? ^
- 'recall@10': 0.6,
? ^
+ 'recall@10': 0.5,
? ^
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.5,
? ^
+ 'recall@5': 0.5,
- 'recall@5': 0.6,
+ 'recall@50': 0.6,
? +
- 'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmp8689_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmp8689_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
E Differing items:
E {'recall@10': 0.5} != {'recall@10': 0.6}
E {'recall@100': 0.8} != {'recall@100': 0.9}
E {'recall@5': 0.5} != {'recall@5': 0.6}
E {'recall@1': 0.2} != {'recall@1': 0.4}
E {'recall@50': 0.6} != {'recall@50': 0.7}
E {'recall@3': 0.5} != {'recall@3': 0.6}
E Full diff:
E {
E - 'recall@1': 0.4,
E ? ^
E + 'recall@1': 0.2,
E ? ^
E - 'recall@10': 0.6,
E ? ^
E + 'recall@10': 0.5,
E ? ^
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.5,
E ? ^
E + 'recall@5': 0.5,
E - 'recall@5': 0.6,
E + 'recall@50': 0.6,
E ? +
E - 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.6, ...} == {'recall@1': ...@3': 0.6, ...}
Omitting 5 identical items, use -vv to show
Differing items:
{'recall@100': 0.8} != {'recall@100': 0.9}
Full diff:
{
'recall@1': 0.4,
'recall@10': 0.6,
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
'recall@3': 0.6,
'recall@5': 0.6,
'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmp1096_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmp1096_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.6, ...} == {'recall@1': ...@3': 0.6, ...}
E Omitting 5 identical items, use -vv to show
E Differing items:
E {'recall@100': 0.8} != {'recall@100': 0.9}
E Full diff:
E {
E 'recall@1': 0.4,
E 'recall@10': 0.6,
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E 'recall@3': 0.6,
E 'recall@5': 0.6,
E 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.4, ...} == {'recall@1': ...@3': 0.6, ...}
Differing items:
{'recall@50': 0.5} != {'recall@50': 0.7}
{'recall@100': 0.7} != {'recall@100': 0.9}
{'recall@5': 0.4} != {'recall@5': 0.6}
{'recall@1': 0.3} != {'recall@1': 0.4}
{'recall@10': 0.4} != {'recall@10': 0.6}
{'recall@3': 0.4} != {'recall@3': 0.6}
Full diff:
{
+ 'recall@1': 0.3,
- 'recall@1': 0.4,
+ 'recall@10': 0.4,
? +
- 'recall@10': 0.6,
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.7,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.4,
? ^
- 'recall@5': 0.6,
? ^
+ 'recall@5': 0.4,
? ^
- 'recall@50': 0.7,
? ^
+ 'recall@50': 0.5,
? ^
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmpd680_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmpd680_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.4, ...} == {'recall@1': ...@3': 0.6, ...}
E Differing items:
E {'recall@50': 0.5} != {'recall@50': 0.7}
E {'recall@100': 0.7} != {'recall@100': 0.9}
E {'recall@5': 0.4} != {'recall@5': 0.6}
E {'recall@1': 0.3} != {'recall@1': 0.4}
E {'recall@10': 0.4} != {'recall@10': 0.6}
E {'recall@3': 0.4} != {'recall@3': 0.6}
E Full diff:
E {
E + 'recall@1': 0.3,
E - 'recall@1': 0.4,
E + 'recall@10': 0.4,
E ? +
E - 'recall@10': 0.6,
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.7,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.4,
E ? ^
E - 'recall@5': 0.6,
E ? ^
E + 'recall@5': 0.4,
E ? ^
E - 'recall@50': 0.7,
E ? ^
E + 'recall@50': 0.5,
E ? ^
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
Differing items:
{'recall@10': 0.5} != {'recall@10': 0.6}
{'recall@1': 0.3} != {'recall@1': 0.4}
{'recall@5': 0.5} != {'recall@5': 0.6}
{'recall@3': 0.5} != {'recall@3': 0.6}
{'recall@50': 0.6} != {'recall@50': 0.7}
{'recall@100': 0.8} != {'recall@100': 0.9}
Full diff:
{
- 'recall@1': 0.4,
? ^
+ 'recall@1': 0.3,
? ^
- 'recall@10': 0.6,
? ^
+ 'recall@10': 0.5,
? ^
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.5,
? ^
+ 'recall@5': 0.5,
- 'recall@5': 0.6,
+ 'recall@50': 0.6,
? +
- 'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmpe231_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmpe231_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
E Differing items:
E {'recall@10': 0.5} != {'recall@10': 0.6}
E {'recall@1': 0.3} != {'recall@1': 0.4}
E {'recall@5': 0.5} != {'recall@5': 0.6}
E {'recall@3': 0.5} != {'recall@3': 0.6}
E {'recall@50': 0.6} != {'recall@50': 0.7}
E {'recall@100': 0.8} != {'recall@100': 0.9}
E Full diff:
E {
E - 'recall@1': 0.4,
E ? ^
E + 'recall@1': 0.3,
E ? ^
E - 'recall@10': 0.6,
E ? ^
E + 'recall@10': 0.5,
E ? ^
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.5,
E ? ^
E + 'recall@5': 0.5,
E - 'recall@5': 0.6,
E + 'recall@50': 0.6,
E ? +
E - 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
Omitting 1 identical items, use -vv to show
Differing items:
{'recall@10': 0.5} != {'recall@10': 0.6}
{'recall@5': 0.5} != {'recall@5': 0.6}
{'recall@100': 0.7} != {'recall@100': 0.9}
{'recall@50': 0.6} != {'recall@50': 0.7}
{'recall@3': 0.5} != {'recall@3': 0.6}
Full diff:
{
'recall@1': 0.4,
- 'recall@10': 0.6,
? ^
+ 'recall@10': 0.5,
? ^
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.7,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.5,
? ^
+ 'recall@5': 0.5,
- 'recall@5': 0.6,
+ 'recall@50': 0.6,
? +
- 'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmp8f64_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmp8f64_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
E Omitting 1 identical items, use -vv to show
E Differing items:
E {'recall@10': 0.5} != {'recall@10': 0.6}
E {'recall@5': 0.5} != {'recall@5': 0.6}
E {'recall@100': 0.7} != {'recall@100': 0.9}
E {'recall@50': 0.6} != {'recall@50': 0.7}
E {'recall@3': 0.5} != {'recall@3': 0.6}
E Full diff:
E {
E 'recall@1': 0.4,
E - 'recall@10': 0.6,
E ? ^
E + 'recall@10': 0.5,
E ? ^
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.7,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.5,
E ? ^
E + 'recall@5': 0.5,
E - 'recall@5': 0.6,
E + 'recall@50': 0.6,
E ? +
E - 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Check failure on line 189 in deeplake/core/vectorstore/deep_memory/test_deepmemory.py
github-actions / JUnit Test Report
test_deepmemory.test_deepmemory_evaluate
AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
Omitting 1 identical items, use -vv to show
Differing items:
{'recall@50': 0.6} != {'recall@50': 0.7}
{'recall@5': 0.5} != {'recall@5': 0.6}
{'recall@100': 0.8} != {'recall@100': 0.9}
{'recall@10': 0.5} != {'recall@10': 0.6}
{'recall@3': 0.5} != {'recall@3': 0.6}
Full diff:
{
'recall@1': 0.4,
- 'recall@10': 0.6,
? ^
+ 'recall@10': 0.5,
? ^
- 'recall@100': 0.9,
? ^
+ 'recall@100': 0.8,
? ^
- 'recall@3': 0.6,
? ^
+ 'recall@3': 0.5,
? ^
+ 'recall@5': 0.5,
- 'recall@5': 0.6,
+ 'recall@50': 0.6,
? +
- 'recall@50': 0.7,
}Raw output
corpus_query_relevances_copy = ('hub://testingacc2/tmpc247_test_deepmemory_test_deepmemory_evaluate', ['0-dimensional biomaterials lack inductive pro...5107', 1]], [['32587939', 1]], ...], 'hub://testingacc2/tmpc247_test_deepmemory_test_deepmemory_evaluate_eval_queries')
questions_embeddings_and_relevances = (array([[-0.01518817, 0.02033963, -0.01228631, ..., -0.00286692,
-0.0079668 , -0.00414979],
[-0.003503...A treatment decreases endoplasmic reticulum stress in response to general endoplasmic reticulum stress markers.', ...])
hub_cloud_dev_token = 'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJpZCI6InRlc3RpbmdhY2MyIiwiYXBpX2tleSI6IjU4Y0tLb1p6UE1BbThPU2RpbTRiZ2tBekhWekt1VUE3MFJpNTNyZUpKRTJuaiJ9.'
@pytest.mark.slow
@pytest.mark.timeout(600)
@pytest.mark.skipif(sys.platform == "win32", reason="Does not run on Windows")
@requires_libdeeplake
def test_deepmemory_evaluate(
corpus_query_relevances_copy,
questions_embeddings_and_relevances,
hub_cloud_dev_token,
):
corpus, _, _, query_path = corpus_query_relevances_copy
(
questions_embeddings,
question_relevances,
queries,
) = questions_embeddings_and_relevances
db = VectorStore(
corpus,
runtime={"tensor_db": True},
token=hub_cloud_dev_token,
)
# when qvs_params is wrong:
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
# embedding_function is not provided in the constructor or in the eval method
with pytest.raises(ValueError):
db.deep_memory.evaluate(
queries=queries,
relevance=question_relevances,
qvs_params={
"log_queries": True,
"branch_name": "wrong_branch",
},
)
recall = db.deep_memory.evaluate(
queries=queries,
embedding=questions_embeddings,
relevance=question_relevances,
qvs_params={
"branch": "queries",
},
)
> assert recall["without model"] == {
"recall@1": 0.4,
"recall@3": 0.6,
"recall@5": 0.6,
"recall@10": 0.6,
"recall@50": 0.7,
"recall@100": 0.9,
}
E AssertionError: assert {'recall@1': ...@3': 0.5, ...} == {'recall@1': ...@3': 0.6, ...}
E Omitting 1 identical items, use -vv to show
E Differing items:
E {'recall@50': 0.6} != {'recall@50': 0.7}
E {'recall@5': 0.5} != {'recall@5': 0.6}
E {'recall@100': 0.8} != {'recall@100': 0.9}
E {'recall@10': 0.5} != {'recall@10': 0.6}
E {'recall@3': 0.5} != {'recall@3': 0.6}
E Full diff:
E {
E 'recall@1': 0.4,
E - 'recall@10': 0.6,
E ? ^
E + 'recall@10': 0.5,
E ? ^
E - 'recall@100': 0.9,
E ? ^
E + 'recall@100': 0.8,
E ? ^
E - 'recall@3': 0.6,
E ? ^
E + 'recall@3': 0.5,
E ? ^
E + 'recall@5': 0.5,
E - 'recall@5': 0.6,
E + 'recall@50': 0.6,
E ? +
E - 'recall@50': 0.7,
E }
deeplake/core/vectorstore/deep_memory/test_deepmemory.py:189: AssertionError
Loading