Add first working versions of flex and highlights layouts

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 // indirect
+	github.com/gammazero/deque v0.2.1 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect
 	github.com/golang/protobuf v1.5.2 // indirect

go.sum

@@ -174,6 +174,8 @@ github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD
 github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
+github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
+github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
 github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

internal/clip/clip.go

@@ -29,6 +29,19 @@ func DotProductFloat32Float(a []float32, b []Float) (float32, error) {
 	return dot, nil
 }
 
+func DotProductFloat32Float32(a []float32, b []float32) (float32, error) {
+	l := len(a)
+	if l != len(b) {
+		return 0, fmt.Errorf("slice lengths do not match, a %d b %d", l, len(b))
+	}
+
+	dot := float32(0)
+	for i := 0; i < l; i++ {
+		dot += a[i] * b[i]
+	}
+	return dot, nil
+}
+
 // Most real world inverse vector norms of embeddings fall
 // within ~500 of 11843, so it's more efficient to store
 // the inverse vector norm as an offset of this number.

internal/geo/geo.go

@@ -101,6 +101,9 @@ func (g *Geo) String() string {
 	if g == nil || !g.config.ReverseGeocode {
 		return "geo reverse geocoding disabled"
 	}
+	if g.gp == nil {
+		return "geo geopackage not loaded"
+	}
 	return "geo using " + g.uri
 }
 

internal/image/database.go

@@ -1,6 +1,7 @@
 package image
 
 import (
+	"context"
 	"embed"
 	"errors"
 	"fmt"
@@ -125,6 +126,17 @@ type TagIdRange struct {
 
 type tagSet map[tag.Id]struct{}
 
+func readEmbedding(stmt *sqlite.Stmt, invnormIndex int, embeddingIndex int) (clip.Embedding, error) {
+	invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(invnormIndex))
+	size := stmt.ColumnLen(embeddingIndex)
+	bytes := make([]byte, size)
+	read := stmt.ColumnBytes(embeddingIndex, bytes)
+	if read != size {
+		return nil, fmt.Errorf("unable to read embedding bytes, expected %d, got %d", size, read)
+	}
+	return clip.FromRaw(bytes, invnorm), nil
+}
+
 func (tags *tagSet) Add(id tag.Id) {
 	(*tags)[id] = struct{}{}
 }
@@ -1467,6 +1479,115 @@ func (source *Database) List(dirs []string, options ListOptions) <-chan InfoList
 	return out
 }
 
+func (source *Database) ListWithEmbeddings(dirs []string, options ListOptions) <-chan InfoEmb {
+	out := make(chan InfoEmb, 1000)
+	go func() {
+		defer metrics.Elapsed("list infos sqlite")()
+
+		conn := source.pool.Get(context.TODO())
+		defer source.pool.Put(conn)
+
+		sql := ""
+
+		sql += `
+			SELECT infos.id, width, height, orientation, color, created_at_unix, created_at_tz_offset, latitude, longitude, inv_norm, embedding
+			FROM infos
+			INNER JOIN clip_emb ON clip_emb.file_id = id
+		`
+
+		sql += `
+			WHERE path_prefix_id IN (
+				SELECT id
+				FROM prefix
+				WHERE `
+
+		for i := range dirs {
+			sql += `str LIKE ? `
+			if i < len(dirs)-1 {
+				sql += "OR "
+			}
+		}
+
+		sql += `
+			)
+		`
+
+		switch options.OrderBy {
+		case None:
+		case DateAsc:
+			sql += `
+			ORDER BY created_at_unix ASC
+			`
+		case DateDesc:
+			sql += `
+			ORDER BY created_at_unix DESC
+			`
+		default:
+			panic("Unsupported listing order")
+		}
+
+		if options.Limit > 0 {
+			sql += `
+				LIMIT ?
+			`
+		}
+
+		sql += ";"
+
+		stmt := conn.Prep(sql)
+		defer stmt.Reset()
+
+		bindIndex := 1
+
+		for _, dir := range dirs {
+			stmt.BindText(bindIndex, dir+"%")
+			bindIndex++
+		}
+
+		if options.Limit > 0 {
+			stmt.BindInt64(bindIndex, (int64)(options.Limit))
+		}
+
+		for {
+			if exists, err := stmt.Step(); err != nil {
+				log.Printf("Error listing files: %s\n", err.Error())
+			} else if !exists {
+				break
+			}
+
+			var info InfoEmb
+			info.Id = (ImageId)(stmt.ColumnInt64(0))
+
+			info.Width = stmt.ColumnInt(1)
+			info.Height = stmt.ColumnInt(2)
+			info.Orientation = Orientation(stmt.ColumnInt(3))
+			info.Color = (uint32)(stmt.ColumnInt64(4))
+
+			unix := stmt.ColumnInt64(5)
+			timezoneOffset := stmt.ColumnInt(6)
+
+			info.DateTime = time.Unix(unix, 0).In(time.FixedZone("", timezoneOffset*60))
+
+			if stmt.ColumnType(7) == sqlite.TypeNull || stmt.ColumnType(8) == sqlite.TypeNull {
+				info.LatLng = NaNLatLng()
+			} else {
+				info.LatLng = s2.LatLngFromDegrees(stmt.ColumnFloat(7), stmt.ColumnFloat(8))
+			}
+
+			emb, err := readEmbedding(stmt, 9, 10)
+			if err != nil {
+				log.Printf("Error reading embedding: %s\n", err.Error())
+			}
+			info.Embedding = emb
+
+			out <- info
+		}
+
+		close(out)
+	}()
+	return out
+}
+
 func (source *Database) GetImageEmbedding(id ImageId) (clip.Embedding, error) {
 	conn := source.pool.Get(nil)
 	defer source.pool.Put(conn)
@@ -1553,19 +1674,15 @@ func (source *Database) ListEmbeddings(dirs []string, options ListOptions) <-cha
 			}
 
 			id := (ImageId)(stmt.ColumnInt64(0))
-			invnorm := uint16(clip.InvNormMean + stmt.ColumnInt64(1))
-
-			size := stmt.ColumnLen(2)
-			bytes := make([]byte, size)
-			read := stmt.ColumnBytes(2, bytes)
-			if read != size {
-				log.Printf("Error reading embedding: buffer underrun, expected %d actual %d bytes\n", size, read)
+			emb, err := readEmbedding(stmt, 1, 2)
+			if err != nil {
+				log.Printf("Error reading embedding: %s\n", err.Error())
 				continue
 			}
 
 			out <- EmbeddingsResult{
 				Id:        id,
-				Embedding: clip.FromRaw(bytes, invnorm),
+				Embedding: emb,
 			}
 		}
 

internal/image/source.go

@@ -80,6 +80,11 @@ type SimilarityInfo struct {
 	Similarity float32
 }
 
+type InfoEmb struct {
+	SourcedInfo
+	Embedding clip.Embedding
+}
+
 func SimilarityInfosToSourcedInfos(sinfos <-chan SimilarityInfo) <-chan SourcedInfo {
 	out := make(chan SourcedInfo)
 	go func() {
@@ -398,6 +403,23 @@ func (source *Source) ListInfos(dirs []string, options ListOptions) <-chan Sourc
 	return out
 }
 
+func (source *Source) ListInfosEmb(dirs []string, options ListOptions) <-chan InfoEmb {
+	for i := range dirs {
+		dirs[i] = filepath.FromSlash(dirs[i])
+	}
+	out := make(chan InfoEmb, 1000)
+	go func() {
+		defer metrics.Elapsed("list infos embedded")()
+
+		infos := source.database.ListWithEmbeddings(dirs, options)
+		for info := range infos {
+			out <- info
+		}
+		close(out)
+	}()
+	return out
+}
+
 func (source *Source) ListInfosWithExistence(dirs []string, options ListOptions) <-chan SourcedInfo {
 	for i := range dirs {
 		dirs[i] = filepath.FromSlash(dirs[i])

internal/layout/common.go

@@ -17,13 +17,15 @@ import (
 type Type string
 
 const (
-	Album    Type = "ALBUM"
-	Timeline Type = "TIMELINE"
-	Square   Type = "SQUARE"
-	Wall     Type = "WALL"
-	Map      Type = "MAP"
-	Search   Type = "SEARCH"
-	Strip    Type = "STRIP"
+	Album      Type = "ALBUM"
+	Timeline   Type = "TIMELINE"
+	Square     Type = "SQUARE"
+	Wall       Type = "WALL"
+	Map        Type = "MAP"
+	Search     Type = "SEARCH"
+	Strip      Type = "STRIP"
+	Highlights Type = "HIGHLIGHTS"
+	Flex       Type = "FLEX"
 )
 
 type Order int