Skip to content

Commit

Permalink
feat(indexers): add External Identifier to map with ARR indexers (#1534)
Browse files Browse the repository at this point in the history 
* feat(indexers): add External Identifier to map with ARR indexers

* fix: web build

* fix: tests

* feat: set identifier for manual processing
  • Loading branch information
@zze0s
zze0s committed May 4, 2024
1 parent ad6ef22 commit 0016228
Show file tree
Hide file tree
Showing 26 changed files with 254 additions and 197 deletions.
7 changes: 4 additions & 3 deletions internal/action/exec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,10 @@ func Test_service_execCmd(t *testing.T) {
TorrentName: "This is a test",
TorrentTmpFile: "tmp-10000",
Indexer: domain.IndexerMinimal{
ID: 0,
Name: "Mock Indexer",
Identifier: "mock",
ID: 0,
Name: "Mock Indexer",
Identifier: "mock",
IdentifierExternal: "Mock Indexer",
},
},
action: &domain.Action{
Expand Down
2 changes: 1 addition & 1 deletion internal/action/lidarr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (s *service) lidarr(ctx context.Context, action *domain.Action, release dom
DownloadUrl: release.DownloadURL,
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer.Identifier,
Indexer: release.Indexer.IdentifierExternal,
DownloadClientId: externalClientId,
DownloadClient: externalClient,
DownloadProtocol: string(release.Protocol),
Expand Down
2 changes: 1 addition & 1 deletion internal/action/radarr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (s *service) radarr(ctx context.Context, action *domain.Action, release dom
DownloadUrl: release.DownloadURL,
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer.Identifier,
Indexer: release.Indexer.IdentifierExternal,
DownloadClientId: externalClientId,
DownloadClient: externalClient,
DownloadProtocol: string(release.Protocol),
Expand Down
2 changes: 1 addition & 1 deletion internal/action/readarr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (s *service) readarr(ctx context.Context, action *domain.Action, release do
DownloadUrl: release.DownloadURL,
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer.Identifier,
Indexer: release.Indexer.IdentifierExternal,
DownloadClientId: externalClientId,
DownloadClient: externalClient,
DownloadProtocol: string(release.Protocol),
Expand Down
2 changes: 1 addition & 1 deletion internal/action/sonarr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (s *service) sonarr(ctx context.Context, action *domain.Action, release dom
DownloadUrl: release.DownloadURL,
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer.Identifier,
Indexer: release.Indexer.IdentifierExternal,
DownloadClientId: externalClientId,
DownloadClient: externalClient,
DownloadProtocol: string(release.Protocol),
Expand Down
2 changes: 1 addition & 1 deletion internal/action/whisparr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (s *service) whisparr(ctx context.Context, action *domain.Action, release d
DownloadUrl: release.DownloadURL,
MagnetUrl: release.MagnetURI,
Size: int64(release.Size),
Indexer: release.Indexer.Identifier,
Indexer: release.Indexer.IdentifierExternal,
DownloadClientId: externalClientId,
DownloadClient: externalClient,
DownloadProtocol: string(release.Protocol),
Expand Down
2 changes: 1 addition & 1 deletion internal/announce/announce.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (a *announceProcessor) processQueue(queue chan string) {
continue
}

rls := domain.NewRelease(domain.IndexerMinimal{ID: a.indexer.ID, Name: a.indexer.Name, Identifier: a.indexer.Identifier})
rls := domain.NewRelease(domain.IndexerMinimal{ID: a.indexer.ID, Name: a.indexer.Name, Identifier: a.indexer.Identifier, IdentifierExternal: a.indexer.IdentifierExternal})
rls.Protocol = domain.ReleaseProtocol(a.indexer.Protocol)

// on lines matched
Expand Down
9 changes: 6 additions & 3 deletions internal/database/feed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func (r *FeedRepo) FindByID(ctx context.Context, id int) (*domain.Feed, error) {
"f.id",
"i.id",
"i.identifier",
"i.identifier_external",
"i.name",
"f.name",
"f.type",
Expand Down Expand Up @@ -66,7 +67,7 @@ func (r *FeedRepo) FindByID(ctx context.Context, id int) (*domain.Feed, error) {

var apiKey, cookie, settings sql.NullString

if err := row.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
if err := row.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.IdentifierExternal, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

Expand All @@ -91,6 +92,7 @@ func (r *FeedRepo) FindByIndexerIdentifier(ctx context.Context, indexer string)
"f.id",
"i.id",
"i.identifier",
"i.identifier_external",
"i.name",
"f.name",
"f.type",
Expand Down Expand Up @@ -123,7 +125,7 @@ func (r *FeedRepo) FindByIndexerIdentifier(ctx context.Context, indexer string)

var apiKey, cookie, settings sql.NullString

if err := row.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
if err := row.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.IdentifierExternal, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

Expand All @@ -146,6 +148,7 @@ func (r *FeedRepo) Find(ctx context.Context) ([]domain.Feed, error) {
"f.id",
"i.id",
"i.identifier",
"i.identifier_external",
"i.name",
"f.name",
"f.type",
Expand Down Expand Up @@ -185,7 +188,7 @@ func (r *FeedRepo) Find(ctx context.Context) ([]domain.Feed, error) {
var apiKey, cookie, lastRunData, settings sql.NullString
var lastRun sql.NullTime

if err := rows.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &lastRun, &lastRunData, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
if err := rows.Scan(&f.ID, &f.Indexer.ID, &f.Indexer.Identifier, &f.Indexer.IdentifierExternal, &f.Indexer.Name, &f.Name, &f.Type, &f.Enabled, &f.URL, &f.Interval, &f.Timeout, &f.MaxAge, &apiKey, &cookie, &lastRun, &lastRunData, &settings, &f.CreatedAt, &f.UpdatedAt); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

Expand Down
45 changes: 23 additions & 22 deletions internal/database/indexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,19 +36,16 @@ func (r *IndexerRepo) Store(ctx context.Context, indexer domain.Indexer) (*domai
}

queryBuilder := r.db.squirrel.
Insert("indexer").Columns("enabled", "name", "identifier", "implementation", "base_url", "settings").
Values(indexer.Enabled, indexer.Name, indexer.Identifier, indexer.Implementation, indexer.BaseURL, settings).
Insert("indexer").Columns("enabled", "name", "identifier", "identifier_external", "implementation", "base_url", "settings").
Values(indexer.Enabled, indexer.Name, indexer.Identifier, indexer.IdentifierExternal, indexer.Implementation, indexer.BaseURL, settings).
Suffix("RETURNING id").RunWith(r.db.handler)

// return values
var retID int64

if err = queryBuilder.QueryRowContext(ctx).Scan(&retID); err != nil {
err = queryBuilder.QueryRowContext(ctx).Scan(&indexer.ID)
if err != nil {
return nil, errors.Wrap(err, "error executing query")
}

indexer.ID = retID

return &indexer, nil
}

Expand All @@ -62,6 +59,7 @@ func (r *IndexerRepo) Update(ctx context.Context, indexer domain.Indexer) (*doma
Update("indexer").
Set("enabled", indexer.Enabled).
Set("name", indexer.Name).
Set("identifier_external", indexer.IdentifierExternal).
Set("base_url", indexer.BaseURL).
Set("settings", settings).
Set("updated_at", time.Now().Format(time.RFC3339)).
Expand All @@ -81,7 +79,7 @@ func (r *IndexerRepo) Update(ctx context.Context, indexer domain.Indexer) (*doma

func (r *IndexerRepo) List(ctx context.Context) ([]domain.Indexer, error) {
queryBuilder := r.db.squirrel.
Select("id", "enabled", "name", "identifier", "implementation", "base_url", "settings").
Select("id", "enabled", "name", "identifier", "identifier_external", "implementation", "base_url", "settings").
From("indexer").
OrderBy("name ASC")

Expand All @@ -102,16 +100,17 @@ func (r *IndexerRepo) List(ctx context.Context) ([]domain.Indexer, error) {
for rows.Next() {
var f domain.Indexer

var implementation, baseURL sql.NullString
var identifierExternal, implementation, baseURL sql.Null[string]
var settings string
var settingsMap map[string]string

if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &f.Identifier, &implementation, &baseURL, &settings); err != nil {
if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &f.Identifier, &identifierExternal, &implementation, &baseURL, &settings); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

f.Implementation = implementation.String
f.BaseURL = baseURL.String
f.IdentifierExternal = identifierExternal.V
f.Implementation = implementation.V
f.BaseURL = baseURL.V

if err = json.Unmarshal([]byte(settings), &settingsMap); err != nil {
return nil, errors.Wrap(err, "error unmarshal settings")
Expand All @@ -130,7 +129,7 @@ func (r *IndexerRepo) List(ctx context.Context) ([]domain.Indexer, error) {

func (r *IndexerRepo) FindByID(ctx context.Context, id int) (*domain.Indexer, error) {
queryBuilder := r.db.squirrel.
Select("id", "enabled", "name", "identifier", "implementation", "base_url", "settings").
Select("id", "enabled", "name", "identifier", "identifier_external", "implementation", "base_url", "settings").
From("indexer").
Where(sq.Eq{"id": id})

Expand All @@ -146,17 +145,18 @@ func (r *IndexerRepo) FindByID(ctx context.Context, id int) (*domain.Indexer, er

var i domain.Indexer

var implementation, baseURL, settings sql.NullString
var identifierExternal, implementation, baseURL, settings sql.Null[string]

if err := row.Scan(&i.ID, &i.Enabled, &i.Name, &i.Identifier, &implementation, &baseURL, &settings); err != nil {
if err := row.Scan(&i.ID, &i.Enabled, &i.Name, &i.Identifier, &identifierExternal, &implementation, &baseURL, &settings); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

i.Implementation = implementation.String
i.BaseURL = baseURL.String
i.IdentifierExternal = identifierExternal.V
i.Implementation = implementation.V
i.BaseURL = baseURL.V

var settingsMap map[string]string
if err = json.Unmarshal([]byte(settings.String), &settingsMap); err != nil {
if err = json.Unmarshal([]byte(settings.V), &settingsMap); err != nil {
return nil, errors.Wrap(err, "error unmarshal settings")
}

Expand All @@ -167,7 +167,7 @@ func (r *IndexerRepo) FindByID(ctx context.Context, id int) (*domain.Indexer, er

func (r *IndexerRepo) FindByFilterID(ctx context.Context, id int) ([]domain.Indexer, error) {
queryBuilder := r.db.squirrel.
Select("id", "enabled", "name", "identifier", "base_url", "settings").
Select("id", "enabled", "name", "identifier", "identifier_external", "base_url", "settings").
From("indexer").
Join("filter_indexer ON indexer.id = filter_indexer.indexer_id").
Where(sq.Eq{"filter_indexer.filter_id": id})
Expand All @@ -190,17 +190,18 @@ func (r *IndexerRepo) FindByFilterID(ctx context.Context, id int) ([]domain.Inde

var settings string
var settingsMap map[string]string
var baseURL sql.NullString
var identifierExternal, baseURL sql.Null[string]

if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &f.Identifier, &baseURL, &settings); err != nil {
if err := rows.Scan(&f.ID, &f.Enabled, &f.Name, &f.Identifier, &identifierExternal, &baseURL, &settings); err != nil {
return nil, errors.Wrap(err, "error scanning row")
}

if err = json.Unmarshal([]byte(settings), &settingsMap); err != nil {
return nil, errors.Wrap(err, "error unmarshal settings")
}

f.BaseURL = baseURL.String
f.IdentifierExternal = identifierExternal.V
f.BaseURL = baseURL.V
f.Settings = settingsMap

indexers = append(indexers, f)
Expand Down
25 changes: 16 additions & 9 deletions internal/database/postgres_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ CREATE TABLE users
CREATE TABLE indexer
(
id SERIAL PRIMARY KEY,
identifier TEXT,
implementation TEXT,
base_url TEXT,
enabled BOOLEAN,
name TEXT NOT NULL,
settings TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
id SERIAL PRIMARY KEY,
identifier TEXT,
identifier_external TEXT,
implementation TEXT,
base_url TEXT,
enabled BOOLEAN,
name TEXT NOT NULL,
settings TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE (identifier)
);
Expand Down Expand Up @@ -877,5 +878,11 @@ ALTER TABLE filter
`,
`ALTER TABLE action
ADD COLUMN first_last_piece_prio BOOLEAN DEFAULT false;
`,
`ALTER TABLE indexer
ADD COLUMN identifier_external TEXT;
UPDATE indexer
SET identifier_external = name;
`,
}
25 changes: 16 additions & 9 deletions internal/database/sqlite_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ CREATE TABLE users
CREATE TABLE indexer
(
id INTEGER PRIMARY KEY,
identifier TEXT,
implementation TEXT,
base_url TEXT,
enabled BOOLEAN,
name TEXT NOT NULL,
settings TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
id INTEGER PRIMARY KEY,
identifier TEXT,
identifier_external TEXT,
implementation TEXT,
base_url TEXT,
enabled BOOLEAN,
name TEXT NOT NULL,
settings TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE (identifier)
);
Expand Down Expand Up @@ -1515,5 +1516,11 @@ ALTER TABLE filter
`,
`ALTER TABLE action
ADD COLUMN first_last_piece_prio BOOLEAN DEFAULT false;
`,
`ALTER TABLE indexer
ADD COLUMN identifier_external TEXT;
UPDATE indexer
SET identifier_external = name;
`,
}
59 changes: 31 additions & 28 deletions internal/domain/indexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,43 @@ type IndexerRepo interface {
}

type Indexer struct {
ID int64 `json:"id"`
Name string `json:"name"`
Identifier string `json:"identifier"`
Enabled bool `json:"enabled"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Settings map[string]string `json:"settings,omitempty"`
ID int64 `json:"id"`
Name string `json:"name"`
Identifier string `json:"identifier"`
IdentifierExternal string `json:"identifier_external"`
Enabled bool `json:"enabled"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Settings map[string]string `json:"settings,omitempty"`
}

type IndexerMinimal struct {
ID int `json:"id"`
Name string `json:"name"`
Identifier string `json:"identifier"`
ID int `json:"id"`
Name string `json:"name"`
Identifier string `json:"identifier"`
IdentifierExternal string `json:"identifier_external"`
}

type IndexerDefinition struct {
ID int `json:"id,omitempty"`
Name string `json:"name"`
Identifier string `json:"identifier"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Enabled bool `json:"enabled"`
Description string `json:"description"`
Language string `json:"language"`
Privacy string `json:"privacy"`
Protocol string `json:"protocol"`
URLS []string `json:"urls"`
Supports []string `json:"supports"`
Settings []IndexerSetting `json:"settings,omitempty"`
SettingsMap map[string]string `json:"-"`
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
Newznab *Newznab `json:"newznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
ID int `json:"id,omitempty"`
Name string `json:"name"`
Identifier string `json:"identifier"`
IdentifierExternal string `json:"identifier_external"`
Implementation string `json:"implementation"`
BaseURL string `json:"base_url,omitempty"`
Enabled bool `json:"enabled"`
Description string `json:"description"`
Language string `json:"language"`
Privacy string `json:"privacy"`
Protocol string `json:"protocol"`
URLS []string `json:"urls"`
Supports []string `json:"supports"`
Settings []IndexerSetting `json:"settings,omitempty"`
SettingsMap map[string]string `json:"-"`
IRC *IndexerIRC `json:"irc,omitempty"`
Torznab *Torznab `json:"torznab,omitempty"`
Newznab *Newznab `json:"newznab,omitempty"`
RSS *FeedSettings `json:"rss,omitempty"`
}

type IndexerImplementation string
Expand Down

0 comments on commit 0016228

Please sign in to comment.