TO BE PUSHED ONCE test on niko and complex-test are OK as well as DB migration Feat/sim search b2f backup before rebase

QA/py/pg_files/upgrade_prod.sql

@@ -2219,6 +2219,29 @@ UPDATE alembic_version SET version_num='4e25988b1e56' WHERE alembic_version.vers
 ALTER TABLE users ADD COLUMN orcid VARCHAR(20) DEFAULT NULL;
 UPDATE alembic_version SET version_num='0a3132f436fb' WHERE alembic_version.version_num = '4e25988b1e56';
 
+-- Running upgrade 0a3132f436fb -> a9dd3c62b7b0
+CREATE TABLE obj_cnn_features_vector (
+    objcnnid BIGINT NOT NULL,
+    features VECTOR(50),
+    PRIMARY KEY (objcnnid),
+    FOREIGN KEY(objcnnid) REFERENCES obj_head (objid) ON DELETE CASCADE
+);
+
+INSERT INTO obj_cnn_features_vector (objcnnid, features)
+        SELECT objcnnid, ARRAY[cnn01, cnn02, cnn03, cnn04, cnn05, cnn06, cnn07, cnn08, cnn09, cnn10,
+        cnn11, cnn12, cnn13, cnn14, cnn15, cnn16, cnn17, cnn18, cnn19, cnn20,
+        cnn21, cnn22, cnn23, cnn24, cnn25, cnn26, cnn27, cnn28, cnn29, cnn30,
+        cnn31, cnn32, cnn33, cnn34, cnn35, cnn36, cnn37, cnn38, cnn39, cnn40,
+        cnn41, cnn42, cnn43, cnn44, cnn45, cnn46, cnn47, cnn48, cnn49, cnn50]::vector
+        FROM obj_cnn_features;
+
+DROP TABLE obj_cnn_features;
+
+UPDATE alembic_version SET version_num='a9dd3c62b7b0' WHERE alembic_version.version_num = '0a3132f436fb';
+
+COMMIT;
+
+
 ------- Leave on tail
 
 ALTER TABLE alembic_version REPLICA IDENTITY FULL;

QA/py/requirements.txt

@@ -9,6 +9,8 @@ sqlalchemy_views==0.3.1
 # When SQLAlchemy moves to 2.0 we'll be able to use psycopg3
 #psycopg[binary]==3.0.8
 psycopg2-binary==2.9.3
+# pgvector for similarity search
+pgvector==0.2.4
 # Pillow needs manual testing
 Pillow==8.1.0
 # astral needs pytz

QA/py/tests/test_similarity_search.py

@@ -0,0 +1,98 @@
+import logging
+import pytest
+import pandas as pd
+import numpy as np
+
+from starlette import status
+
+from tests.credentials import ADMIN_AUTH
+from tests.jobs import get_job_and_wait_until_ok, api_check_job_ok
+from tests.test_objectset_query import _prj_query
+from tests.test_classification import classify_all
+
+from BO.Prediction import DeepFeatures
+
+from API_operations.CRUD.ObjectParents import SamplesService
+
+copepod_id = 25828
+entomobryomorpha_id = 25835
+crustacea = 12846
+
+
+def similarity_scores(target_id, distances_to_target):
+    return [round(1 - d / max(distances_to_target), 4) for d in distances_to_target]
+
+def test_similarity_search(database, fastapi, caplog):
+    caplog.set_level(logging.ERROR)
+    from tests.test_import import test_import
+
+    prj_id = test_import(database, caplog, "Test Similarity Search")
+
+    obj_ids = _prj_query(fastapi, ADMIN_AUTH, prj_id)
+    assert len(obj_ids) == 8
+
+    # Prepare dummy features
+    features = list()
+    for i, oi in enumerate(obj_ids):
+        features.append([(i + 1) * 0.1] * 50)
+    features_df = pd.DataFrame(features, index=obj_ids)
+
+    target_id = obj_ids[0]
+    distances_to_target = [np.linalg.norm(features_df.loc[target_id] - features_df.loc[oi]) for oi in obj_ids]
+
+    # Test similarity search without features
+    url = f"/object_set/{prj_id}/similarity_search"
+    req = {
+        "project_id": prj_id,
+        "target_id": target_id,
+    }
+    filters = {}
+    req_and_filters = {"filters": filters, "request": req}
+    rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters)
+
+    assert rsp.status_code == status.HTTP_200_OK
+    assert rsp.json()["message"] == "Missing CNN features, please select a feature extractor"
+    assert rsp.json()["neighbor_ids"] == []
+    assert rsp.json()["sim_scores"] == []
+
+    # Insert dummy features
+    with SamplesService() as sce:
+        n_inserts = DeepFeatures.save(sce.session, features_df)
+        assert n_inserts == 8
+        sce.session.commit()
+
+    # Test similarity search with features
+    url = f"/object_set/{prj_id}/similarity_search"
+    req = {
+        "project_id": prj_id,
+        "target_id": target_id,
+    }
+    filters = {}
+    req_and_filters = {"filters": filters, "request": req}
+    rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters)
+
+    assert rsp.status_code == status.HTTP_200_OK
+    assert rsp.json()["message"] == "Success"
+    assert rsp.json()["neighbor_ids"] == obj_ids
+    assert rsp.json()["sim_scores"] == similarity_scores(target_id, distances_to_target)
+
+    # Set different taxo ids
+    classify_all(fastapi, obj_ids[0:3], copepod_id)
+    classify_all(fastapi, obj_ids[3:5], entomobryomorpha_id)
+    classify_all(fastapi, obj_ids[5:8], crustacea)
+    taxo_ids_to_filter = [copepod_id, entomobryomorpha_id]
+
+    # Test similarity search with filters on taxo
+    url = f"/object_set/{prj_id}/similarity_search"
+    req = {
+        "project_id": prj_id,
+        "target_id": target_id,
+    }
+    filters = {"taxo": ",".join([str(taxo_id) for taxo_id in taxo_ids_to_filter])}
+    req_and_filters = {"filters": filters, "request": req}
+    rsp = fastapi.post(url, headers=ADMIN_AUTH, json=req_and_filters)
+
+    assert rsp.status_code == status.HTTP_200_OK
+    assert rsp.json()["message"] == "Success"
+    assert rsp.json()["neighbor_ids"] == obj_ids[0:5]
+    assert rsp.json()["sim_scores"] == similarity_scores(target_id, distances_to_target[0:5])

documentation/query_update_cnnfeature_vector_migrate_0124.log

@@ -0,0 +1,12 @@
+ecotaxa4=# CREATE TABLE obj_cnn_features_vector AS SELECT objcnnid, ARRAY[cnn01, cnn02, cnn03, cnn04, cnn05, cnn06, cnn07, cnn08, cnn09, cnn10, cnn11, cnn12, cnn13, cnn14, cnn15, cnn16, cnn17, cnn18, cnn19, cnn20, cnn21, cnn22, cnn23, cnn24, cnn25, cnn26, cnn27, cnn28, cnn29, cnn30, cnn31, cnn32, cnn33, cnn34, cnn35, cnn36, cnn37, cnn38, cnn39, cnn40, cnn41, cnn42, cnn43, cnn44, cnn45, cnn46, cnn47, cnn48, cnn49, cnn50] as features FROM obj_cnn_features;
+
+SELECT 146710153
+Time: 377084.366 ms (06:17.084)
+
+ecotaxa4=# UPDATE obj_cnn_features_vector SET features=features::vector;
+UPDATE 146710153
+Time: 725130.979 ms (12:05.131)
+
+ecotaxa4=# ALTER TABLE obj_cnn_features_vector ADD PRIMARY KEY (objcnnid);
+ALTER TABLE
+Time: 363022.969 ms (06:03.023)

py/API_models/filters.py

@@ -73,6 +73,10 @@ class ProjectFiltersDict(TypedDict, total=False):
     filt_last_annot: Optional[str]
     """ Coma-separated list of annotator, i.e. person who validated the classification
         in last. """
+    seed_object_id: Optional[str]
+    """
+    Target objid for similarity search
+    """
 
 
 class _ProjectFilters2Model(DescriptiveModel):
@@ -225,6 +229,11 @@ class _ProjectFilters2Model(DescriptiveModel):
         description="Coma-separated list of annotators, i.e. persons who validated the classification in last.",
         example="34,67",
     )
+    seed_object_id = Field(
+        title="Seed object id",
+        description="Target objid for similarity search",
+        example="1234",
+    )
 
     class Config:
         schema_extra = {

py/API_models/simsearch.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# This file is part of Ecotaxa, see license.md in the application root directory for license informations.
+# Copyright (C) 2015-2020  Picheral, Colin, Irisson (UPMC-CNRS)
+#
+from typing import List, Optional, Dict
+
+from helpers.pydantic import BaseModel, Field
+
+class SimilaritySearchReq(BaseModel):
+    """
+    Similarity search request.
+    """
+
+    project_id: int = Field(
+        title="Project Id",
+        description="The destination project, in which we want to find similar objects.",
+    )
+
+    target_id: int = Field(
+        title="Target Id",
+        description="The object we want to find similar objects for.",
+    )
+
+    class Config:
+        schema_extra = {
+            "title": "Similarity Search Request",
+            "description": "How to find similar objects, in details.",
+            "example": {
+                "project_id": 3426,
+                "target_id": 1040,
+            },
+        }
+
+
+class SimilaritySearchRsp(BaseModel):
+    """
+    Similarity search response.
+    """
+
+    neighbor_ids: List[int] = Field(
+        title="Neighbor IDs",
+        description="The list of similar objects.",
+    )
+
+    sim_scores: List[float] = Field(
+        title="Similarity Scores",
+        description="The list of similarity scores.",
+    )
+
+    message: Optional[str] = Field(
+        title="Message",
+        description="A message to the user.",
+    )
+
+    class Config:
+        schema_extra = {
+            "title": "Similarity Search Response",
+            "description": "The list of similar objects.",
+            "example": {
+                "neighbor_ids": [1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047],
+                "sim_scores": [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3],
+                "message": "Success",
+            },
+        }

py/API_operations/FeatureExtraction.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# This file is part of Ecotaxa, see license.md in the application root directory for license informations.
+# Copyright (C) 2015-2021  Picheral, Colin, Irisson (UPMC-CNRS)
+#
+# Extract deep DeepFeatures on a project.
+#
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+from BO.Rights import RightsBO, Action
+from BO.User import UserIDT
+from DB.Project import ProjectIDT, Project
+from FS.MachineLearningModels import SavedModels
+from FS.Vault import Vault
+from helpers.DynamicLogs import get_logger, LogsSwitcher
+from .helpers.JobService import JobServiceOnProjectBase, ArgsDict
+
+logger = get_logger(__name__)
+
+
+class FeatureExtractionForProject(JobServiceOnProjectBase):
+    """ """
+
+    JOB_TYPE = "FeatureExtraction"
+
+    def __init__(self, prj_id: ProjectIDT) -> None:
+        super().__init__(prj_id)
+        self.vault = Vault(self.config.vault_dir())
+        self.models_dir = SavedModels(self.config)
+
+    def run(self, current_user_id: UserIDT) -> None:
+        """
+        Initial creation, do security and consistency checks, then create the job.
+        """
+        _user, _project = RightsBO.user_wants(
+            self.session, current_user_id, Action.ANNOTATE, self.prj_id
+        )
+        # TODO: more checks, e.g. deep features models consistency
+        # Security OK, create pending job
+        self.create_job(self.JOB_TYPE, current_user_id)
+
+    def do_background(self) -> None:
+        """
+        Background part of the job.
+        """
+        with LogsSwitcher(self):
+            self.ensure_deep_features_job()
+
+    def ensure_deep_features_job(self) -> None:
+        ...
+

py/API_operations/GPU_FeatureExtraction.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# This file is part of Ecotaxa, see license.md in the application root directory for license informations.
+# Copyright (C) 2015-2021  Picheral, Colin, Irisson (UPMC-CNRS)
+#
+# Extract DeepFeatures on a project.
+#
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+from BO.Prediction import DeepFeatures
+from BO.Rights import RightsBO, Action
+from DB.Project import ProjectIDT, Project
+from ML.Deep_features_extractor import DeepFeaturesExtractor
+from helpers.DynamicLogs import get_logger
+from .helpers.JobService import JobServiceBase
+from .FeatureExtraction import FeatureExtractionForProject
+from helpers.Timer import CodeTimer
+
+logger = get_logger(__name__)
+
+# Remove job type for base class, so during run the flow ends here
+FeatureExtractionForProject.JOB_TYPE = ""
+
+class GPUFeatureExtractionForProject(FeatureExtractionForProject):
+    """
+    Part of the feature extraction which needs special HW and libs.
+    """
+
+    JOB_TYPE = "FeatureExtraction"
+
+    DEEP_EXTRACT_CHUNK = 10000
+
+    def ensure_deep_features_job(self) -> None:
+        """
+        Ensure that deep features are present for given project.
+        """
+        proj_id = self.prj_id
+        model_name = self.prj.cnn_network_id
+
+        msg = self._ensure_deep_features_for(proj_id, model_name)
+        done_infos = {"message": msg}
+
+        self.set_job_result(errors=[], infos=done_infos)
+        return
+
+    def _ensure_deep_features_for(self, proj_id: ProjectIDT, model_name: str) -> str:
+        """
+        Ensure that deep features are present for given project.
+        """
+        # Get data i.e objects ID and images from the project
+        ids_and_images = DeepFeatures.find_missing(self.ro_session, proj_id)
+        if len(ids_and_images) == 0:
+            return "All CNN present for %d" % proj_id
+
+        # Do reasonable chunks so we can see logs...
+        nb_rows = 0
+        extractor = DeepFeaturesExtractor(self.vault, self.models_dir)
+        while len(ids_and_images) > 0:
+            chunk = {}
+            for objid, img in ids_and_images.items():
+                chunk[objid] = img
+                if len(chunk) >= self.DEEP_EXTRACT_CHUNK:
+                    break
+            for objid in chunk.keys():
+                del ids_and_images[objid]
+
+            # Call feature extractor
+            features = extractor.run(chunk, model_name)
+
+            # Save CNN
+            with CodeTimer("Saving %d new CNN " % self.DEEP_EXTRACT_CHUNK, logger):
+                nb_rows += DeepFeatures.save(self.session, features)
+            self.session.commit()
+
+        return "OK, %d CNN features computed and written for %d" % (nb_rows, proj_id)