Add ability to search for similar images

.gitignore

@@ -17,6 +17,7 @@
 /assets/facenet
 /assets/nasnet
 /assets/nsfw
+/assets/clip
 /assets/static/build/
 /package-lock.json
 /frontend/tests_output

compose.armv7.yaml

@@ -53,6 +53,7 @@ services:
       PHOTOPRISM_DISABLE_PLACES: "false"             # Disables reverse geocoding and maps
       PHOTOPRISM_DISABLE_EXIFTOOL: "false"           # Don't create ExifTool JSON files for improved metadata extraction
       PHOTOPRISM_DISABLE_TENSORFLOW: "false"         # Don't use TensorFlow for image classification
+      PHOTOPRISM_DISABLE_CLIP: "true"                # Disables indexing photo embeddings using CLIP for similarity search
       PHOTOPRISM_DETECT_NSFW: "false"                # Flag photos as private that MAY be offensive (requires TensorFlow)
       PHOTOPRISM_UPLOAD_NSFW: "false"                # Allows uploads that may be offensive
       PHOTOPRISM_DARKTABLE_PRESETS: "false"          # Enables Darktable presets and disables concurrent RAW conversion

compose.ci.yaml

@@ -49,6 +49,7 @@ services:
       PHOTOPRISM_DISABLE_PLACES: "false"      # disables reverse geocoding and maps
       PHOTOPRISM_DISABLE_EXIFTOOL: "false"    # disables creating JSON metadata sidecar files with ExifTool
       PHOTOPRISM_DISABLE_TENSORFLOW: "false"  # disables all features depending on TensorFlow
+      PHOTOPRISM_DISABLE_CLIP: "true"                # Disables indexing photo embeddings using CLIP for similarity search
       PHOTOPRISM_DETECT_NSFW: "false"         # automatically flags photos as private that MAY be offensive (requires TensorFlow)
       PHOTOPRISM_UPLOAD_NSFW: "false"         # allow uploads that may be offensive
       PHOTOPRISM_RAW_PRESETS: "false"         # enables applying user presets when converting RAW images (reduces performance)

compose.latest.yaml

@@ -48,6 +48,7 @@ services:
       PHOTOPRISM_DISABLE_PLACES: "false"      # disables reverse geocoding and maps
       PHOTOPRISM_DISABLE_EXIFTOOL: "false"    # disables creating JSON metadata sidecar files with ExifTool
       PHOTOPRISM_DISABLE_TENSORFLOW: "false"  # disables all features depending on TensorFlow
+      PHOTOPRISM_DISABLE_CLIP: "true"                # Disables indexing photo embeddings using CLIP for similarity search
       PHOTOPRISM_DETECT_NSFW: "false"         # automatically flags photos as private that MAY be offensive (requires TensorFlow)
       PHOTOPRISM_UPLOAD_NSFW: "false"         # allows uploads that MAY be offensive (no effect without TensorFlow)
       PHOTOPRISM_RAW_PRESETS: "false"         # enables applying user presets when converting RAW images (reduces performance)

frontend/src/component/photo/cards.vue

@@ -141,6 +141,12 @@
 
           <div class="pa-3 card-details">
             <div>
+              <template v-if="filter.order === 'similar'">
+                  <button :title="$gettext('Similarity score')" @click.exact="downloadFile(index)">
+                    <i>image_search</i>
+                    {{ photo.SimilarityScore?.toFixed(3) }}
+                  </button>
+              </template>
               <h3 class="body-2 mb-2" :title="photo.Title">
                 <button class="action-title-edit" :data-uid="photo.UID" @click.exact="isSharedView ? openPhoto(index) : editPhoto(index)">
                   {{ photo.Title | truncate(80) }}

frontend/src/dialog/photo/edit.vue

@@ -70,6 +70,14 @@
           </template>
         </v-tab>
 
+        <v-tab id="tab-similar" ripple>
+          <v-icon v-if="$vuetify.breakpoint.smAndDown" :title="$gettext('Similar photos')">image_search</v-icon>
+          <template v-else>
+            <v-icon :size="18" :left="!rtl" :right="rtl">image_search</v-icon>
+            <translate key="Similar photos">Similar photos</translate>
+          </template>
+        </v-tab>
+
         <v-tab v-if="$config.feature('edit')" id="tab-info" ripple>
           <v-icon>settings</v-icon>
         </v-tab>
@@ -91,6 +99,10 @@
             <p-tab-photo-files :key="uid" :model="model" :uid="uid" @close="close"></p-tab-photo-files>
           </v-tab-item>
 
+          <v-tab-item lazy>
+            <p-tab-photo-similar :key="uid" :model="model" :uid="uid" @close="close"></p-tab-photo-similar>
+          </v-tab-item>
+
           <v-tab-item v-if="$config.feature('edit')" lazy>
             <p-tab-photo-info :key="uid" :model="model" :uid="uid" @close="close"></p-tab-photo-info>
           </v-tab-item>
@@ -105,6 +117,7 @@ import PhotoDetails from "./edit/details.vue";
 import PhotoLabels from "./edit/labels.vue";
 import PhotoPeople from "./edit/people.vue";
 import PhotoFiles from "./edit/files.vue";
+import PhotoSimilar from "./edit/similar.vue";
 import PhotoInfo from "./edit/info.vue";
 import Event from "pubsub-js";
 
@@ -115,6 +128,7 @@ export default {
     "p-tab-photo-labels": PhotoLabels,
     "p-tab-photo-people": PhotoPeople,
     "p-tab-photo-files": PhotoFiles,
+    "p-tab-photo-similar": PhotoSimilar,
     "p-tab-photo-info": PhotoInfo,
   },
   props: {

frontend/src/dialog/photo/edit/similar.vue

@@ -0,0 +1,55 @@
+<template>
+  <div class="p-tab p-tab-photo-similar">
+    <v-layout class="pa-2-md-and-up" row wrap align-top fill-height>
+      <v-flex class="pa-2" xs12 md2 xxl1 fill-height>
+        <p-photo-preview :model="model"></p-photo-preview>
+      </v-flex>
+      <v-flex class="pa-2" xs12 md10 xxl11 fill-width fill-height>
+        <p class="subheading">
+          Similar photos
+        </p>
+        <p-photo-cards :photos="results" :filter="filter"></p-photo-cards>
+      </v-flex>
+    </v-layout>
+  </div>
+</template>
+
+<script>
+import Photo from "model/photo";
+
+export default {
+  name: "PTabPhotoSimilar",
+  props: {
+    model: {
+      type: Object,
+      default: () => {},
+    },
+    uid: {
+      type: String,
+      default: "",
+    },
+  },
+  data() {
+    return {
+      loading: true,
+      results: [],
+      filter: {
+        order: "similar",
+      }
+    };
+  },
+  computed: {},
+  methods: {
+  },
+  created() {
+    this.loading = true;
+    Photo.searchSimilar(this.model.UID)
+      .then((response) => {
+        this.results = response.models;
+      })
+      .finally(() => {
+          this.loading = false;
+        });
+  }
+};
+</script>

frontend/src/model/photo.js

@@ -1213,6 +1213,22 @@ export class Photo extends RestModel {
 
     return results.concat(response.models);
   }
+
+  static searchSimilar(uid) {
+    return Api.get(`/photos/${uid}/similar`).then((resp) => {
+      let count = resp.data ? resp.data.length : 0;
+
+      resp.models = [];
+
+      if (count > 0) {
+        for (let i = 0; i < resp.data.length; i++) {
+          resp.models.push(new this(resp.data[i]));
+        }
+      }
+
+      return Promise.resolve(resp);
+    });
+  }
 }
 
 export default Photo;

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.7
 	github.com/leandro-lugaresi/hub v1.1.1
 	github.com/leonelquinteros/gotext v1.5.2
-	github.com/lib/pq v1.10.9 // indirect
+	github.com/lib/pq v1.10.9
 	github.com/lucasb-eyer/go-colorful v1.2.0
 	github.com/mandykoh/prism v0.35.2
 	github.com/manifoldco/promptui v0.9.0
@@ -59,10 +59,7 @@ require github.com/olekukonko/tablewriter v0.0.5
 
 require github.com/google/uuid v1.6.0
 
-require (
-	github.com/chzyer/readline v1.5.1 // indirect
-	github.com/zitadel/oidc v1.13.5
-)
+require github.com/chzyer/readline v1.5.1 // indirect
 
 require github.com/gabriel-vasile/mimetype v1.4.3
 
@@ -71,8 +68,6 @@ require (
 	golang.org/x/time v0.5.0
 )
 
-require github.com/go-ldap/ldap/v3 v3.4.6
-
 require (
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/common v0.52.2
@@ -82,10 +77,12 @@ require github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aa
 
 require golang.org/x/text v0.14.0
 
-require github.com/pquerna/otp v1.4.0
+require (
+	github.com/DataIntelligenceCrew/go-faiss v0.2.0
+	github.com/pquerna/otp v1.4.0
+)
 
 require (
-	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boombuler/barcode v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -96,15 +93,10 @@ require (
 	github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect
 	github.com/dsoprea/go-utility/v2 v2.0.0-20221003172846-a3e1774ef349 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/gorilla/schema v1.2.1 // indirect
-	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -121,13 +113,9 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
-	github.com/zitadel/logging v0.5.0 // indirect
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
-	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
-	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )