Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RFC: Alias search #875

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/NzbDrone.Core/Datastore/Migration/035_release_group_alias.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;

namespace NzbDrone.Core.Datastore.Migration
{
[Migration(35)]
public class release_group_alias : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Alter.Table("Albums").AddColumn("Aliases").AsString().WithDefaultValue("[]");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,16 @@ public virtual Decision IsSatisfiedBy(RemoteAlbum remoteAlbum, SearchCriteriaBas
{
return Decision.Accept();
}
if (Parser.Parser.CleanArtistName(singleAlbumSpec.AlbumTitle) != Parser.Parser.CleanArtistName(remoteAlbum.ParsedAlbumInfo.AlbumTitle))

if (!remoteAlbum.Albums.Any(x => x.Title == singleAlbumSpec.AlbumTitle))
{
_logger.Debug("Album does not match searched album title, skipping.");
return Decision.Reject("Wrong album");
}

if (!remoteAlbum.ParsedAlbumInfo.AlbumTitle.Any())
if (remoteAlbum.Albums.Count > 1)
{
_logger.Debug("Full discography result during single album search, skipping.");
_logger.Debug("Discography result during single album search, skipping.");
return Decision.Reject("Full artist pack");
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,28 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;

namespace NzbDrone.Core.IndexerSearch.Definitions
{
public class AlbumSearchCriteria : SearchCriteriaBase
{
public string AlbumTitle { get; set; }
public List<string> AlbumAliases { get; set; }
public int AlbumYear { get; set; }
public string Disambiguation { get; set; }

public string AlbumQuery => GetQueryTitle($"{AlbumTitle}{(Disambiguation.IsNullOrWhiteSpace() ? string.Empty : $"+{Disambiguation}")}");
public string AlbumQuery => GetQueryTitle(AddDisambiguation(AlbumTitle));

public List<string> AlbumQueries => OrderQueries(AlbumTitle, AlbumAliases)
.Select(x => GetQueryTitle(AddDisambiguation(x)))
.Distinct()
.ToList();

private string AddDisambiguation(string term)
{
return Disambiguation.IsNullOrWhiteSpace() ? term : $"{term}+{Disambiguation}";
}

public override string ToString()
{
Expand Down
34 changes: 34 additions & 0 deletions src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public abstract class SearchCriteriaBase
private static readonly Regex SpecialCharacter = new Regex(@"[`'’.]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex NonWord = new Regex(@"[\W]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex BeginningThe = new Regex(@"^the\s", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex IsAllWord = new Regex(@"^[\sA-Za-z0-9_`'’.&:-]*$", RegexOptions.IgnoreCase | RegexOptions.Compiled);

public virtual bool MonitoredEpisodesOnly { get; set; }
public virtual bool UserInvokedSearch { get; set; }
Expand All @@ -22,6 +23,39 @@ public abstract class SearchCriteriaBase
public List<Track> Tracks { get; set; }

public string ArtistQuery => GetQueryTitle(Artist.Name);
public List<string> ArtistQueries => OrderQueries(Artist.Metadata.Value.Name, Artist.Metadata.Value.Aliases);

protected List<string> OrderQueries(string title, List<string> aliases)
{
var result = new List<string>();

// find the primary search term. This will be title if there are no special characters in the title,
// otherwise the first alias with no special characters
if (IsAllWord.IsMatch(title))
{
result.Add(title);
}
else
{
result.Add(aliases.FirstOrDefault(x => IsAllWord.IsMatch(x)) ?? title);
result.Add(title);
}

// insert remaining aliases
result.AddRange(aliases.Except(result));

return result;
}

protected List<List<string>> GetQueryTiers(List<string> titles)
{
var result = new List<List<string>>();

var queries = titles.Select(GetQueryTitle).Distinct();
result.Add(queries.Take(1).ToList());
result.Add(queries.Skip(1).ToList());
return result;
}

public static string GetQueryTitle(string title)
{
Expand Down
1 change: 1 addition & 0 deletions src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public List<DownloadDecision> AlbumSearch(Album album, bool missingOnly, bool us
var searchSpec = Get<AlbumSearchCriteria>(artist, new List<Album> { album }, userInvokedSearch, interactiveSearch);

searchSpec.AlbumTitle = album.Title;
searchSpec.AlbumAliases = album.Aliases;
if (album.ReleaseDate.HasValue)
{
searchSpec.AlbumYear = album.ReleaseDate.Value.Year;
Expand Down
71 changes: 59 additions & 12 deletions src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common.Extensions;
Expand Down Expand Up @@ -69,49 +70,95 @@ public virtual IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria

if (SupportsAudioSearch)
{
AddAudioPageableRequests(pageableRequests, searchCriteria,
NewsnabifyTitle($"&artist={searchCriteria.ArtistQuery}&album={searchCriteria.AlbumQuery}"));
AddAlbumRequests(pageableRequests, searchCriteria, "&artist={0}&album={1}", AddAudioPageableRequests);
}

if (SupportsSearch)
{
pageableRequests.AddTier();

pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}+{searchCriteria.AlbumQuery}")));
AddAlbumRequests(pageableRequests, searchCriteria, "&q={0}+{1}", AddSearchPageableRequests);
}

return pageableRequests;
}

private void AddAlbumRequests(IndexerPageableRequestChain pageableRequests, AlbumSearchCriteria searchCriteria, string paramFormat, Action<IndexerPageableRequestChain, SearchCriteriaBase, string> AddRequests)
{
var albumQuery = searchCriteria.AlbumQueries[0];
var artistQuery = searchCriteria.ArtistQueries[0];

// search using standard name
pageableRequests.AddTier();
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery, albumQuery)));

// using artist alias
pageableRequests.AddTier();
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt, albumQuery)));
}

// using album alias
pageableRequests.AddTier();
foreach (var albumAlt in searchCriteria.AlbumQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery, albumAlt)));
}

// using aliases for both
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
foreach (var albumAlt in searchCriteria.AlbumQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt, albumAlt)));
}
}
}

public virtual IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();

if (SupportsAudioSearch)
{
AddAudioPageableRequests(pageableRequests, searchCriteria,
NewsnabifyTitle($"&artist={searchCriteria.ArtistQuery}"));
AddArtistRequests(pageableRequests, searchCriteria, "&artist={0}", AddAudioPageableRequests);
}

if (SupportsSearch)
{
pageableRequests.AddTier();

pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search",
NewsnabifyTitle($"&q={searchCriteria.ArtistQuery}")));
AddArtistRequests(pageableRequests, searchCriteria, "&q={0}", AddSearchPageableRequests);
}

return pageableRequests;
}

private void AddArtistRequests(IndexerPageableRequestChain pageableRequests, SearchCriteriaBase searchCriteria, string paramFormat, Action<IndexerPageableRequestChain, SearchCriteriaBase, string> AddRequests)
{
var artistQuery = searchCriteria.ArtistQueries[0];

// search using standard name
pageableRequests.AddTier();
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistQuery)));

// using artist alias
pageableRequests.AddTier();
foreach (var artistAlt in searchCriteria.ArtistQueries.Skip(1))
{
AddRequests(pageableRequests, searchCriteria, NewsnabifyTitle(string.Format(paramFormat, artistAlt)));
}
}

private void AddAudioPageableRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string parameters)
{
chain.AddTier();

chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "music", $"&q={parameters}"));
}

private void AddSearchPageableRequests(IndexerPageableRequestChain chain, SearchCriteriaBase searchCriteria, string parameters)
{
chain.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", parameters));
}

private IEnumerable<IndexerRequest> GetPagedRequests(int maxPages, IEnumerable<int> categories, string searchType, string parameters)
{
if (categories.Empty())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ namespace NzbDrone.Core.MetadataSource.SkyHook.Resource
{
public class AlbumResource
{
public AlbumResource()
{
Aliases = new List<string>();
}

public List<string> Aliases { get; set; }
public string ArtistId { get; set; }
public List<ArtistResource> Artists { get; set; }
public string Disambiguation { get; set; }
Expand Down
2 changes: 1 addition & 1 deletion src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Exceptions;
Expand Down Expand Up @@ -300,6 +299,7 @@ private static Album MapAlbum(AlbumResource resource, Dictionary<string, ArtistM
album.ForeignAlbumId = resource.Id;
album.OldForeignAlbumIds = resource.OldIds;
album.Title = resource.Title;
album.Aliases = resource.Aliases;
album.Overview = resource.Overview;
album.Disambiguation = resource.Disambiguation;
album.ReleaseDate = resource.ReleaseDate;
Expand Down
4 changes: 4 additions & 0 deletions src/NzbDrone.Core/Music/Album.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public class Album : ModelBase, IEquatable<Album>
{
public Album()
{
Aliases = new List<string>();
Genres = new List<string>();
Images = new List<MediaCover.MediaCover>();
Links = new List<Links>();
Expand All @@ -28,6 +29,7 @@ public Album()
public string ForeignAlbumId { get; set; }
public List<string> OldForeignAlbumIds { get; set; }
public string Title { get; set; }
public List<string> Aliases { get; set; }
public string Overview { get; set; }
public string Disambiguation { get; set; }
public DateTime? ReleaseDate { get; set; }
Expand Down Expand Up @@ -80,6 +82,7 @@ public bool Equals(Album other)
ForeignAlbumId == other.ForeignAlbumId &&
(OldForeignAlbumIds?.SequenceEqual(other.OldForeignAlbumIds) ?? true) &&
Title == other.Title &&
(Aliases?.SequenceEqual(other.Aliases) ?? true) &&
Overview == other.Overview &&
Disambiguation == other.Disambiguation &&
ReleaseDate == other.ReleaseDate &&
Expand Down Expand Up @@ -123,6 +126,7 @@ public override int GetHashCode()
hash = hash * 23 + ForeignAlbumId.GetHashCode();
hash = hash * 23 + OldForeignAlbumIds?.GetHashCode() ?? 0;
hash = hash * 23 + Title?.GetHashCode() ?? 0;
hash = hash * 23 + Aliases?.GetHashCode() ?? 0;
hash = hash * 23 + Overview?.GetHashCode() ?? 0;
hash = hash * 23 + Disambiguation?.GetHashCode() ?? 0;
hash = hash * 23 + ReleaseDate?.GetHashCode() ?? 0;
Expand Down
9 changes: 5 additions & 4 deletions src/NzbDrone.Core/Music/AlbumService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ public interface IAlbumService
List<Album> GetAlbumsForRefresh(int artistMetadataId, IEnumerable<string> foreignIds);
Album AddAlbum(Album newAlbum);
Album FindById(string foreignId);
Album FindByTitle(int artistId, string title);
Album FindByTitleInexact(int artistId, string title);
Album FindByTitle(int artistMetadataId, string title);
Album FindByTitleInexact(int artistMetadataId, string title);
List<Album> GetCandidates(int artistId, string title);
void DeleteAlbum(int albumId, bool deleteFiles);
List<Album> GetAllAlbums();
Expand Down Expand Up @@ -76,9 +76,9 @@ public Album FindById(string lidarrId)
return _albumRepository.FindById(lidarrId);
}

public Album FindByTitle(int artistId, string title)
public Album FindByTitle(int artistMetadataId, string title)
{
return _albumRepository.FindByTitle(artistId, title);
return _albumRepository.FindByTitle(artistMetadataId, title);
}

private List<Tuple<Func<Album, string, double>, string>> AlbumScoringFunctions(string title, string cleanTitle)
Expand All @@ -87,6 +87,7 @@ private List<Tuple<Func<Album, string, double>, string>> AlbumScoringFunctions(s
var scoringFunctions = new List<Tuple<Func<Album, string, double>, string>> {
tc((a, t) => a.CleanTitle.FuzzyMatch(t), cleanTitle),
tc((a, t) => a.Title.FuzzyMatch(t), title),
tc((a, t) => a.Aliases.Any() ? a.Aliases.Select(x => x.CleanArtistName().FuzzyMatch(t)).Max() : 0, cleanTitle),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveBracketsAndContents().CleanArtistName()),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveAfterDash().CleanArtistName()),
tc((a, t) => a.CleanTitle.FuzzyMatch(t), title.RemoveBracketsAndContents().RemoveAfterDash().CleanArtistName()),
Expand Down
3 changes: 2 additions & 1 deletion src/NzbDrone.Core/Music/ArtistService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ public List<Tuple<Func<Artist, string, double>, string>> ArtistScoringFunctions(
Func< Func<Artist, string, double>, string, Tuple<Func<Artist, string, double>, string>> tc = Tuple.Create;
var scoringFunctions = new List<Tuple<Func<Artist, string, double>, string>> {
tc((a, t) => a.CleanName.FuzzyMatch(t), cleanTitle),
tc((a, t) => a.Name.FuzzyMatch(t), title),
tc((a, t) => a.Metadata.Value.Name.FuzzyMatch(t), title),
tc((a, t) => a.Metadata.Value.Aliases.Any() ? a.Metadata.Value.Aliases.Select(x => x.CleanArtistName().FuzzyMatch(t)).Max() : 0, cleanTitle)
};

if (title.StartsWith("The ", StringComparison.CurrentCultureIgnoreCase))
Expand Down
1 change: 1 addition & 0 deletions src/NzbDrone.Core/Music/RefreshAlbumService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ protected override UpdateResult UpdateEntity(Album local, Album remote)
local.LastInfoSync = DateTime.UtcNow;
local.CleanTitle = remote.CleanTitle;
local.Title = remote.Title ?? "Unknown";
local.Aliases = remote.Aliases;
local.Overview = remote.Overview.IsNullOrWhiteSpace() ? local.Overview : remote.Overview;
local.Disambiguation = remote.Disambiguation;
local.AlbumType = remote.AlbumType;
Expand Down
3 changes: 2 additions & 1 deletion src/NzbDrone.Core/NzbDrone.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@
<Compile Include="Datastore\Migration\031_add_artistmetadataid_constraint.cs" />
<Compile Include="Datastore\Migration\032_old_ids_and_artist_alias.cs" />
<Compile Include="Datastore\Migration\033_download_propers_config.cs" />
<Compile Include="Datastore\Migration\035_release_group_alias.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationDbFactory.cs" />
Expand Down Expand Up @@ -1355,4 +1356,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>
7 changes: 5 additions & 2 deletions src/NzbDrone.Core/Parser/Parser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,11 @@ public static ParsedAlbumInfo ParseAlbumTitleWithSearchCriteria(string title, Ar

simpleTitle = CleanTorrentSuffixRegex.Replace(simpleTitle, string.Empty);

var escapedArtist = Regex.Escape(artist.Name.RemoveAccent()).Replace(@"\ ", @"[\W_]");
var escapedAlbums = string.Join("|", album.Select(s => Regex.Escape(s.Title.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");
var artistAliases = new [] { artist.Name }.Concat(artist.Metadata.Value.Aliases);
var escapedArtist = string.Join("|", artistAliases.Select(x => Regex.Escape(x.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");

var albumAliases = album.Select(x => x.Title).Concat(album.SelectMany(x => x.Aliases));
var escapedAlbums = string.Join("|", albumAliases.Select(s => Regex.Escape(s.RemoveAccent())).ToList()).Replace(@"\ ", @"[\W_]");

var releaseRegex = new Regex(@"^(\W*|\b)(?<artist>" + escapedArtist + @")(\W*|\b).*(\W*|\b)(?<album>" + escapedAlbums + @")(\W*|\b)", RegexOptions.IgnoreCase);

Expand Down
2 changes: 1 addition & 1 deletion src/NzbDrone.Core/Parser/ParsingService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public List<Album> GetAlbums(ParsedAlbumInfo parsedAlbumInfo, Artist artist, Sea
if (albumInfo == null)
{
_logger.Debug("Trying inexact album match for {0}", parsedAlbumInfo.AlbumTitle);
albumInfo = _albumService.FindByTitleInexact(artist.Id, parsedAlbumInfo.AlbumTitle);
albumInfo = _albumService.FindByTitleInexact(artist.ArtistMetadataId, parsedAlbumInfo.AlbumTitle);
}

if (albumInfo != null)
Expand Down