Ordering (#260)

This PR adds ordering capability in the API and web UI. There is also some refactoring of the web UI that has significantly improved performance.

To get proper ordering (particularly by seeders/leechers), the index must be reprocessed.

I'm aware that certain combinations of filters and orders can be slow, but for most performance is acceptable. For example, ordering a large filtered result by size (ascending) seems very slow, though descending seems okay - I've experimented with a few indexing and database tweaks without much luck and so decided to leave as-is, it can possibly be addressed later or it may be an inherent limitation of Postgres.

Taskfile.yml

@@ -5,8 +5,8 @@ tasks:
     cmds:
       - go generate ./...
       - go run ./internal/dev gorm gen
-      - go run ./internal/gql/enums/gen/genenums.go
       - go run ./internal/torznab/gencategories/gencategories.go
+      - go run ./internal/gql/enums/gen/genenums.go
       - go run github.com/99designs/gqlgen generate --config ./internal/gql/gqlgen.yml
       - protoc --go_out=. ./internal/protobuf/bitmagnet.proto
       - go run github.com/vektra/mockery/v2

graphql/fragments/Torrent.graphql

@@ -4,7 +4,6 @@ fragment Torrent on Torrent {
   infoHash
   name
   size
-  private
   filesStatus
   filesCount
   hasFilesInfo

graphql/fragments/TorrentContent.graphql

@@ -28,6 +28,9 @@ fragment TorrentContent on TorrentContent {
   videoModifier
   videoResolution
   videoSource
+  seeders
+  leechers
+  publishedAt
   createdAt
   updatedAt
 }

graphql/queries/TorrentContentSearch.graphql

@@ -3,9 +3,10 @@
 query TorrentContentSearch(
   $query: SearchQueryInput
   $facets: TorrentContentFacetsInput
+  $orderBy: [TorrentContentOrderByInput!]
 ) {
   torrentContent {
-    search(query: $query, facets: $facets) {
+    search(query: $query, facets: $facets, orderBy: $orderBy) {
       ...TorrentContentSearchResult
     }
   }

graphql/schema/enums.graphqls

@@ -145,3 +145,15 @@ enum VideoSource {
   WEBRip
   BluRay
 }
+
+enum TorrentContentOrderBy {
+  Relevance
+  PublishedAt
+  UpdatedAt
+  Size
+  Files
+  Seeders
+  Leechers
+  Name
+  InfoHash
+}

graphql/schema/models.graphqls

@@ -2,7 +2,6 @@ type Torrent {
   infoHash: Hash20!
   name: String!
   size: Int!
-  private: Boolean!
   hasFilesInfo: Boolean!
   singleFile: Boolean
   extension: String
@@ -56,6 +55,9 @@ type TorrentContent {
   video3d: Video3d
   videoModifier: VideoModifier
   releaseGroup: String
+  seeders: Int
+  leechers: Int
+  publishedAt: DateTime!
   createdAt: DateTime!
   updatedAt: DateTime!
 }

graphql/schema/query.graphqls

@@ -26,6 +26,7 @@ type TorrentContentQuery {
   search(
     query: SearchQueryInput
     facets: TorrentContentFacetsInput
+    orderBy: [TorrentContentOrderByInput!]
   ): TorrentContentSearchResult!
 }
 

graphql/schema/search.graphqls

@@ -157,3 +157,8 @@ type TorrentContentSearchResult {
   items: [TorrentContent!]!
   aggregations: TorrentContentAggregations!
 }
+
+input TorrentContentOrderByInput {
+  field: TorrentContentOrderBy!
+  descending: Boolean
+}

internal/database/query/options.go

@@ -4,12 +4,10 @@ import (
 	"context"
 	"github.com/bitmagnet-io/bitmagnet/internal/database/cache"
 	"github.com/bitmagnet-io/bitmagnet/internal/database/dao"
-	"github.com/bitmagnet-io/bitmagnet/internal/database/fts"
 	"github.com/bitmagnet-io/bitmagnet/internal/maps"
 	"gorm.io/gen/field"
 	"gorm.io/gorm/clause"
 	"gorm.io/gorm/schema"
-	"strings"
 )
 
 type Option = func(ctx OptionBuilder) (OptionBuilder, error)
@@ -74,36 +72,8 @@ func RequireJoin(names ...string) Option {
 }
 
 func QueryString(str string) Option {
-	query := fts.AppQueryToTsquery(str)
 	return func(ctx OptionBuilder) (OptionBuilder, error) {
-		if len(query) == 0 {
-			return ctx.Select(clause.Expr{
-				SQL: "0 AS " + queryStringRankField,
-			}), nil
-		}
-		c, err := GenCriteria(func(ctx DbContext) (Criteria, error) {
-			return DbCriteria{
-				Sql: strings.Join([]string{
-					ctx.TableName() + ".tsv @@ ?::tsquery",
-				}, " OR "),
-				Args: []interface{}{
-					query,
-				},
-			}, nil
-		}).Raw(ctx)
-		if err != nil {
-			return ctx, err
-		}
-		ctx = ctx.Scope(func(dao SubQuery) error {
-			dao.UnderlyingDB().Where(c.Query, c.Args...)
-			return nil
-		}).RequireJoin(ctx.TableName()).Select(clause.Expr{
-			SQL: "ts_rank_cd(" + ctx.TableName() + ".tsv, ?::tsquery) AS " + queryStringRankField,
-			Vars: []interface{}{
-				query,
-			},
-		})
-		return ctx, nil
+		return ctx.QueryString(str), nil
 	}
 }
 
@@ -141,12 +111,12 @@ func OrderBy(columns ...clause.OrderByColumn) Option {
 	}
 }
 
-const queryStringRankField = "query_string_rank"
+const QueryStringRankField = "query_string_rank"
 
 func OrderByQueryStringRank() Option {
 	return func(ctx OptionBuilder) (OptionBuilder, error) {
 		return ctx.OrderBy(clause.OrderByColumn{
-			Column:  clause.Column{Name: queryStringRankField},
+			Column:  clause.Column{Name: QueryStringRankField},
 			Desc:    true,
 			Reorder: true,
 		}), nil

internal/database/query/query.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/bitmagnet-io/bitmagnet/internal/database/dao"
+	"github.com/bitmagnet-io/bitmagnet/internal/database/fts"
 	"github.com/bitmagnet-io/bitmagnet/internal/maps"
 	"github.com/bitmagnet-io/bitmagnet/internal/model"
 	"gorm.io/gen"
@@ -207,6 +208,7 @@ type OptionBuilder interface {
 	Table(string) OptionBuilder
 	Join(...TableJoin) OptionBuilder
 	RequireJoin(...string) OptionBuilder
+	QueryString(string) OptionBuilder
 	Scope(...Scope) OptionBuilder
 	Select(...clause.Expr) OptionBuilder
 	OrderBy(...clause.OrderByColumn) OptionBuilder
@@ -239,6 +241,7 @@ type optionBuilder struct {
 	dbContext
 	joins             map[string]TableJoin
 	requiredJoins     maps.InsertMap[string, struct{}]
+	tsquery           string
 	scopes            []Scope
 	selections        []clause.Expr
 	groupBy           []clause.Column
@@ -302,6 +305,11 @@ func (b optionBuilder) RequireJoin(names ...string) OptionBuilder {
 	return b
 }
 
+func (b optionBuilder) QueryString(str string) OptionBuilder {
+	b.tsquery = fts.AppQueryToTsquery(str)
+	return b
+}
+
 func (b optionBuilder) Scope(scopes ...Scope) OptionBuilder {
 	b.scopes = append(b.scopes, scopes...)
 	return b
@@ -318,7 +326,7 @@ func (b optionBuilder) Group(columns ...clause.Column) OptionBuilder {
 }
 
 func (b optionBuilder) OrderBy(columns ...clause.OrderByColumn) OptionBuilder {
-	b.orderBy = append(b.orderBy, columns...)
+	b.orderBy = columns
 	return b
 }
 
@@ -422,6 +430,19 @@ func (b optionBuilder) applySelect(sq SubQuery) error {
 			selectQueryArgs = append(selectQueryArgs, s.Vars...)
 		}
 	}
+	for _, orderBy := range b.orderBy {
+		if orderBy.Column.Name == QueryStringRankField {
+			rankFragment := "0"
+			args := make([]interface{}, 0)
+			if b.tsquery != "" {
+				rankFragment = "ts_rank_cd(" + b.tableName + ".tsv, ?::tsquery)"
+				args = append(args, b.tsquery)
+			}
+			selectQueryParts = append(selectQueryParts, rankFragment+" AS "+QueryStringRankField)
+			selectQueryArgs = append(selectQueryArgs, args...)
+			break
+		}
+	}
 	sq.UnderlyingDB().Select(strings.Join(selectQueryParts, ", "), selectQueryArgs...)
 	return nil
 }
@@ -432,6 +453,9 @@ func (b optionBuilder) applyPre(sq SubQuery) error {
 			return err
 		}
 	}
+	if b.tsquery != "" {
+		sq.UnderlyingDB().Where(b.tableName+".tsv @@ ?::tsquery", b.tsquery)
+	}
 	requiredJoins := b.requiredJoins.Copy()
 	aggC, aggCErr := b.createFacetsFilterCriteria()
 	if aggCErr != nil {
@@ -502,19 +526,8 @@ func applyJoins(sq SubQuery, joins ...TableJoin) {
 
 func (b optionBuilder) applyPost(sq SubQuery) error {
 	if len(b.orderBy) > 0 {
-		orderBy := make([]clause.OrderByColumn, 0, len(b.orderBy))
-		for _, ob := range b.orderBy {
-			if ob.Reorder {
-				orderBy = append([]clause.OrderByColumn{{
-					Column: ob.Column,
-					Desc:   ob.Desc,
-				}}, orderBy...)
-			} else {
-				orderBy = append(orderBy, ob)
-			}
-		}
 		sq.UnderlyingDB().Statement.AddClause(clause.OrderBy{
-			Columns: orderBy,
+			Columns: b.orderBy,
 		})
 	}
 	if b.limit.Valid {

internal/database/search/order_torrent_content.go

@@ -0,0 +1,140 @@
+package search
+
+import (
+	"github.com/bitmagnet-io/bitmagnet/internal/database/query"
+	"github.com/bitmagnet-io/bitmagnet/internal/maps"
+	"github.com/bitmagnet-io/bitmagnet/internal/model"
+	"gorm.io/gorm/clause"
+)
+
+//go:generate go run github.com/abice/go-enum --marshal --names --nocase --nocomments --sql --sqlnullstr --values -f order_torrent_content.go
+
+// TorrentContentOrderBy represents sort orders for torrent content search results
+// ENUM(Relevance, PublishedAt, UpdatedAt, Size, Files, Seeders, Leechers, Name, InfoHash)
+type TorrentContentOrderBy string
+
+// OrderDirection represents sort order directions
+// ENUM(Ascending, Descending)
+type OrderDirection string
+
+func (ob TorrentContentOrderBy) Clauses(direction OrderDirection) []clause.OrderByColumn {
+	desc := direction == OrderDirectionDescending
+	switch ob {
+	case TorrentContentOrderByRelevance:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Name: query.QueryStringRankField,
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderByPublishedAt:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Table: model.TableNameTorrentContent,
+				Name:  "published_at",
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderByUpdatedAt:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Table: model.TableNameTorrentContent,
+				Name:  "updated_at",
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderBySize:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Table: model.TableNameTorrent,
+				Name:  "size",
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderByFiles:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Name: "CASE WHEN " + model.TableNameTorrent + ".files_status = 'single' THEN 1 ELSE COALESCE(" + model.TableNameTorrent + ".files_count, -1) END",
+				Raw:  true,
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderBySeeders:
+		return []clause.OrderByColumn{
+			{
+				Column: clause.Column{
+					Name: model.TableNameTorrentContent + ".seeders IS NULL",
+					Raw:  true,
+				},
+				Desc: !desc,
+			},
+			{
+				Column: clause.Column{
+					Table: model.TableNameTorrentContent,
+					Name:  "seeders",
+				},
+				Desc: desc,
+			},
+		}
+	case TorrentContentOrderByLeechers:
+		return []clause.OrderByColumn{
+			{
+				Column: clause.Column{
+					Name: model.TableNameTorrentContent + ".leechers IS NULL",
+					Raw:  true,
+				},
+				Desc: !desc,
+			},
+			{
+				Column: clause.Column{
+					Table: model.TableNameTorrentContent,
+					Name:  "leechers",
+				},
+				Desc: desc,
+			},
+		}
+	case TorrentContentOrderByName:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Table: model.TableNameTorrent,
+				Name:  "name",
+			},
+			Desc: desc,
+		}}
+	case TorrentContentOrderByInfoHash:
+		return []clause.OrderByColumn{{
+			Column: clause.Column{
+				Table: model.TableNameTorrentContent,
+				Name:  "info_hash",
+			},
+			Desc: desc,
+		}}
+	default:
+		return []clause.OrderByColumn{}
+	}
+}
+
+type TorrentContentFullOrderBy maps.InsertMap[TorrentContentOrderBy, OrderDirection]
+
+func (fob TorrentContentFullOrderBy) Clauses() []clause.OrderByColumn {
+	im := maps.InsertMap[TorrentContentOrderBy, OrderDirection](fob)
+	clauses := make([]clause.OrderByColumn, 0, im.Len()+2)
+	for _, ob := range im.Entries() {
+		clauses = append(clauses, ob.Key.Clauses(ob.Value)...)
+	}
+	// make ordering alphabetical and deterministic when not already specified:
+	if !im.Has(TorrentContentOrderByName) {
+		clauses = append(clauses, TorrentContentOrderByName.Clauses(OrderDirectionAscending)...)
+	}
+	if !im.Has(TorrentContentOrderByInfoHash) {
+		clauses = append(clauses, TorrentContentOrderByInfoHash.Clauses(OrderDirectionAscending)...)
+	}
+	return clauses
+}
+
+func (fob TorrentContentFullOrderBy) Option() query.Option {
+	return query.Options(
+		query.RequireJoin(model.TableNameTorrent),
+		query.OrderBy(fob.Clauses()...),
+	)
+}