-
Notifications
You must be signed in to change notification settings - Fork 5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
TO BE PUSHED ONCE test on niko and complex-test are OK as well as DB migration Feat/sim search b2f backup before rebase #81
base: master
Are you sure you want to change the base?
Changes from all commits
baedf7e
8ce0f33
add930f
5fac3bd
43ff498
5eaf4b1
53ff797
6ded9bf
0649bea
00180b8
732fa26
5547438
eeb5ecb
57bc453
4899815
ab3ef53
4cf7731
ff780aa
fa2f418
5a49108
5ebce10
511a7fd
64579a4
a9cbb3f
e0475c3
94bc185
a1a9abc
e673612
a98114f
0b8d146
e34102b
7068f40
f48d3a2
3707c44
f1c90fe
fb7b430
b60f723
190852f
cf68094
df5ad9b
73174f9
d8e6fb9
73fbbf6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
import logging | ||
import pytest | ||
import pandas as pd | ||
import numpy as np | ||
|
||
from starlette import status | ||
|
||
from tests.credentials import ADMIN_AUTH | ||
from tests.jobs import get_job_and_wait_until_ok, api_check_job_ok | ||
from tests.test_objectset_query import _prj_query | ||
from tests.test_classification import classify_all | ||
|
||
from BO.Prediction import DeepFeatures | ||
|
||
from API_operations.CRUD.ObjectParents import SamplesService | ||
|
||
copepod_id = 25828 | ||
entomobryomorpha_id = 25835 | ||
crustacea = 12846 | ||
|
||
|
||
def similarity_scores(target_id, distances_to_target): | ||
return [round(1 - d / max(distances_to_target), 4) for d in distances_to_target] | ||
|
||
def test_similarity_search(database, fastapi, caplog): | ||
caplog.set_level(logging.ERROR) | ||
from tests.test_import import test_import | ||
|
||
prj_id = test_import(database, caplog, "Test Similarity Search") | ||
|
||
obj_ids = _prj_query(fastapi, ADMIN_AUTH, prj_id) | ||
assert len(obj_ids) == 8 | ||
|
||
# Prepare dummy features | ||
features = list() | ||
for i, oi in enumerate(obj_ids): | ||
features.append([(i + 1) * 0.1] * 50) | ||
features_df = pd.DataFrame(features, index=obj_ids) | ||
|
||
target_id = obj_ids[0] | ||
distances_to_target = [np.linalg.norm(features_df.loc[target_id] - features_df.loc[oi]) for oi in obj_ids] | ||
|
||
# Test similarity search without features | ||
url = f"/object_set/{prj_id}/similarity_search" | ||
req = { | ||
"project_id": prj_id, | ||
"target_id": target_id, | ||
} | ||
filters = {} | ||
req_and_filters = {"filters": filters, "request": req} | ||
rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters) | ||
|
||
assert rsp.status_code == status.HTTP_200_OK | ||
assert rsp.json()["message"] == "Missing CNN features, please select a feature extractor" | ||
assert rsp.json()["neighbor_ids"] == [] | ||
assert rsp.json()["sim_scores"] == [] | ||
|
||
# Insert dummy features | ||
with SamplesService() as sce: | ||
n_inserts = DeepFeatures.save(sce.session, features_df) | ||
assert n_inserts == 8 | ||
sce.session.commit() | ||
|
||
# Test similarity search with features | ||
url = f"/object_set/{prj_id}/similarity_search" | ||
req = { | ||
"project_id": prj_id, | ||
"target_id": target_id, | ||
} | ||
filters = {} | ||
req_and_filters = {"filters": filters, "request": req} | ||
rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters) | ||
|
||
assert rsp.status_code == status.HTTP_200_OK | ||
assert rsp.json()["message"] == "Success" | ||
assert rsp.json()["neighbor_ids"] == obj_ids | ||
assert rsp.json()["sim_scores"] == similarity_scores(target_id, distances_to_target) | ||
|
||
# Set different taxo ids | ||
classify_all(fastapi, obj_ids[0:3], copepod_id) | ||
classify_all(fastapi, obj_ids[3:5], entomobryomorpha_id) | ||
classify_all(fastapi, obj_ids[5:8], crustacea) | ||
taxo_ids_to_filter = [copepod_id, entomobryomorpha_id] | ||
|
||
# Test similarity search with filters on taxo | ||
url = f"/object_set/{prj_id}/similarity_search" | ||
req = { | ||
"project_id": prj_id, | ||
"target_id": target_id, | ||
} | ||
filters = {"taxo": ",".join([str(taxo_id) for taxo_id in taxo_ids_to_filter])} | ||
req_and_filters = {"filters": filters, "request": req} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pareil que + haut. |
||
rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters) | ||
|
||
assert rsp.status_code == status.HTTP_200_OK | ||
assert rsp.json()["message"] == "Success" | ||
assert rsp.json()["neighbor_ids"] == obj_ids[0:5] | ||
assert rsp.json()["sim_scores"] == similarity_scores(target_id, distances_to_target[0:5]) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
ecotaxa4=# CREATE TABLE obj_cnn_features_vector AS SELECT objcnnid, ARRAY[cnn01, cnn02, cnn03, cnn04, cnn05, cnn06, cnn07, cnn08, cnn09, cnn10, cnn11, cnn12, cnn13, cnn14, cnn15, cnn16, cnn17, cnn18, cnn19, cnn20, cnn21, cnn22, cnn23, cnn24, cnn25, cnn26, cnn27, cnn28, cnn29, cnn30, cnn31, cnn32, cnn33, cnn34, cnn35, cnn36, cnn37, cnn38, cnn39, cnn40, cnn41, cnn42, cnn43, cnn44, cnn45, cnn46, cnn47, cnn48, cnn49, cnn50] as features FROM obj_cnn_features; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ce n'est plus le code de migration actuel, autant virer. |
||
|
||
SELECT 146710153 | ||
Time: 377084.366 ms (06:17.084) | ||
|
||
ecotaxa4=# UPDATE obj_cnn_features_vector SET features=features::vector; | ||
UPDATE 146710153 | ||
Time: 725130.979 ms (12:05.131) | ||
|
||
ecotaxa4=# ALTER TABLE obj_cnn_features_vector ADD PRIMARY KEY (objcnnid); | ||
ALTER TABLE | ||
Time: 363022.969 ms (06:03.023) |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -73,6 +73,10 @@ class ProjectFiltersDict(TypedDict, total=False): | |
filt_last_annot: Optional[str] | ||
""" Coma-separated list of annotator, i.e. person who validated the classification | ||
in last. """ | ||
seed_object_id: Optional[str] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pourquoi c'est dans les filtres, qui sont censés limiter l'affichage? Et pourquoi une str? C'est censé être e.g. |
||
""" | ||
Target objid for similarity search | ||
""" | ||
|
||
|
||
class _ProjectFilters2Model(DescriptiveModel): | ||
|
@@ -225,6 +229,11 @@ class _ProjectFilters2Model(DescriptiveModel): | |
description="Coma-separated list of annotators, i.e. persons who validated the classification in last.", | ||
example="34,67", | ||
) | ||
seed_object_id = Field( | ||
title="Seed object id", | ||
description="Target objid for similarity search", | ||
example="1234", | ||
) | ||
|
||
class Config: | ||
schema_extra = { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
# -*- coding: utf-8 -*- | ||
# This file is part of Ecotaxa, see license.md in the application root directory for license informations. | ||
# Copyright (C) 2015-2020 Picheral, Colin, Irisson (UPMC-CNRS) | ||
# | ||
from typing import List, Optional, Dict | ||
|
||
from helpers.pydantic import BaseModel, Field | ||
|
||
class SimilaritySearchReq(BaseModel): | ||
""" | ||
Similarity search request. | ||
""" | ||
|
||
project_id: int = Field( | ||
title="Project Id", | ||
description="The destination project, in which we want to find similar objects.", | ||
) | ||
|
||
target_id: int = Field( | ||
title="Target Id", | ||
description="The object we want to find similar objects for.", | ||
) | ||
|
||
class Config: | ||
schema_extra = { | ||
"title": "Similarity Search Request", | ||
"description": "How to find similar objects, in details.", | ||
"example": { | ||
"project_id": 3426, | ||
"target_id": 1040, | ||
}, | ||
} | ||
|
||
|
||
class SimilaritySearchRsp(BaseModel): | ||
""" | ||
Similarity search response. | ||
""" | ||
|
||
neighbor_ids: List[int] = Field( | ||
title="Neighbor IDs", | ||
description="The list of similar objects.", | ||
) | ||
|
||
sim_scores: List[float] = Field( | ||
title="Similarity Scores", | ||
description="The list of similarity scores.", | ||
) | ||
|
||
message: Optional[str] = Field( | ||
title="Message", | ||
description="A message to the user.", | ||
) | ||
|
||
class Config: | ||
schema_extra = { | ||
"title": "Similarity Search Response", | ||
"description": "The list of similar objects.", | ||
"example": { | ||
"neighbor_ids": [1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047], | ||
"sim_scores": [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3], | ||
"message": "Success", | ||
}, | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
# -*- coding: utf-8 -*- | ||
# This file is part of Ecotaxa, see license.md in the application root directory for license informations. | ||
# Copyright (C) 2015-2021 Picheral, Colin, Irisson (UPMC-CNRS) | ||
# | ||
# Extract deep DeepFeatures on a project. | ||
# | ||
from typing import Dict, List, Tuple | ||
|
||
import numpy as np | ||
|
||
from BO.Rights import RightsBO, Action | ||
from BO.User import UserIDT | ||
from DB.Project import ProjectIDT, Project | ||
from FS.MachineLearningModels import SavedModels | ||
from FS.Vault import Vault | ||
from helpers.DynamicLogs import get_logger, LogsSwitcher | ||
from .helpers.JobService import JobServiceOnProjectBase, ArgsDict | ||
|
||
logger = get_logger(__name__) | ||
|
||
|
||
class FeatureExtractionForProject(JobServiceOnProjectBase): | ||
""" """ | ||
|
||
JOB_TYPE = "FeatureExtraction" | ||
|
||
def __init__(self, prj_id: ProjectIDT) -> None: | ||
super().__init__(prj_id) | ||
self.vault = Vault(self.config.vault_dir()) | ||
self.models_dir = SavedModels(self.config) | ||
|
||
def run(self, current_user_id: UserIDT) -> None: | ||
""" | ||
Initial creation, do security and consistency checks, then create the job. | ||
""" | ||
_user, _project = RightsBO.user_wants( | ||
self.session, current_user_id, Action.ANNOTATE, self.prj_id | ||
) | ||
# TODO: more checks, e.g. deep features models consistency | ||
# Security OK, create pending job | ||
self.create_job(self.JOB_TYPE, current_user_id) | ||
|
||
def do_background(self) -> None: | ||
""" | ||
Background part of the job. | ||
""" | ||
with LogsSwitcher(self): | ||
self.ensure_deep_features_job() | ||
|
||
def ensure_deep_features_job(self) -> None: | ||
... | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
# -*- coding: utf-8 -*- | ||
# This file is part of Ecotaxa, see license.md in the application root directory for license informations. | ||
# Copyright (C) 2015-2021 Picheral, Colin, Irisson (UPMC-CNRS) | ||
# | ||
# Extract DeepFeatures on a project. | ||
# | ||
from typing import Dict, List, Tuple | ||
|
||
import numpy as np | ||
|
||
from BO.Prediction import DeepFeatures | ||
from BO.Rights import RightsBO, Action | ||
from DB.Project import ProjectIDT, Project | ||
from ML.Deep_features_extractor import DeepFeaturesExtractor | ||
from helpers.DynamicLogs import get_logger | ||
from .helpers.JobService import JobServiceBase | ||
from .FeatureExtraction import FeatureExtractionForProject | ||
from helpers.Timer import CodeTimer | ||
|
||
logger = get_logger(__name__) | ||
|
||
# Remove job type for base class, so during run the flow ends here | ||
FeatureExtractionForProject.JOB_TYPE = "" | ||
|
||
class GPUFeatureExtractionForProject(FeatureExtractionForProject): | ||
""" | ||
Part of the feature extraction which needs special HW and libs. | ||
""" | ||
|
||
JOB_TYPE = "FeatureExtraction" | ||
|
||
DEEP_EXTRACT_CHUNK = 10000 | ||
|
||
def ensure_deep_features_job(self) -> None: | ||
""" | ||
Ensure that deep features are present for given project. | ||
""" | ||
proj_id = self.prj_id | ||
model_name = self.prj.cnn_network_id | ||
|
||
msg = self._ensure_deep_features_for(proj_id, model_name) | ||
done_infos = {"message": msg} | ||
|
||
self.set_job_result(errors=[], infos=done_infos) | ||
return | ||
|
||
def _ensure_deep_features_for(self, proj_id: ProjectIDT, model_name: str) -> str: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Mon IDE dit que c'est un copier/coller de la méthode de même nom dans GPU_Prediction.py, il faudrait l'enlever là-bas. |
||
""" | ||
Ensure that deep features are present for given project. | ||
""" | ||
# Get data i.e objects ID and images from the project | ||
ids_and_images = DeepFeatures.find_missing(self.ro_session, proj_id) | ||
if len(ids_and_images) == 0: | ||
return "All CNN present for %d" % proj_id | ||
|
||
# Do reasonable chunks so we can see logs... | ||
nb_rows = 0 | ||
extractor = DeepFeaturesExtractor(self.vault, self.models_dir) | ||
while len(ids_and_images) > 0: | ||
chunk = {} | ||
for objid, img in ids_and_images.items(): | ||
chunk[objid] = img | ||
if len(chunk) >= self.DEEP_EXTRACT_CHUNK: | ||
break | ||
for objid in chunk.keys(): | ||
del ids_and_images[objid] | ||
|
||
# Call feature extractor | ||
features = extractor.run(chunk, model_name) | ||
|
||
# Save CNN | ||
with CodeTimer("Saving %d new CNN " % self.DEEP_EXTRACT_CHUNK, logger): | ||
nb_rows += DeepFeatures.save(self.session, features) | ||
self.session.commit() | ||
|
||
return "OK, %d CNN features computed and written for %d" % (nb_rows, proj_id) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
C'est la cause du problème de test planté dans l'action. Le filter est (je ne sais pourquoi) censé contenu la cible de la recherche par similarité, et le champ n'est pas rempli ici.