Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove db calls from list threads #9241

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Remove db calls from list threads #9241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f6f4fc
Remove db calls from list threads
Qstick Oct 8, 2023
b279984
fixup! Remove db calls from list threads
Qstick Oct 8, 2023
200be64
fixup! Remove db calls from list threads
Qstick Oct 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListServiceFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
using NUnit.Framework;
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ImportLists.ImportListMovies;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Movies;
using NzbDrone.Core.Test.Framework;

namespace NzbDrone.Core.Test.ImportListTests
Expand Down Expand Up @@ -36,10 +34,6 @@ public void Setup()

_listMovies = Builder<ImportListMovie>.CreateListOfSize(5)
.Build().ToList();

Mocker.GetMock<ISearchForNewMovie>()
.Setup(v => v.MapMovieToTmdbMovie(It.IsAny<MovieMetadata>()))
.Returns<MovieMetadata>(m => new MovieMetadata { TmdbId = m.TmdbId });
}

private void GivenList(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult)
Expand Down Expand Up @@ -135,9 +129,6 @@ public void should_store_movies_if_list_doesnt_fail()

var listResult = Subject.Fetch();
listResult.AnyFailure.Should().BeFalse();

Mocker.GetMock<IImportListMovieService>()
.Verify(v => v.SyncMoviesForList(It.IsAny<List<ImportListMovie>>(), listId), Times.Once());
}

[Test]
Expand All @@ -149,9 +140,6 @@ public void should_not_store_movies_if_list_fails()

var listResult = Subject.Fetch();
listResult.AnyFailure.Should().BeTrue();

Mocker.GetMock<IImportListMovieService>()
.Verify(v => v.SyncMoviesForList(It.IsAny<List<ImportListMovie>>(), listId), Times.Never());
}

[Test]
Expand All @@ -166,9 +154,6 @@ public void should_only_store_movies_for_lists_that_dont_fail()

var listResult = Subject.Fetch();
listResult.AnyFailure.Should().BeTrue();

Mocker.GetMock<IImportListMovieService>()
.Verify(v => v.SyncMoviesForList(It.IsAny<List<ImportListMovie>>(), passedListId), Times.Once());
}

[Test]
Expand Down
18 changes: 15 additions & 3 deletions src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ImportLists.ImportExclusions;
using NzbDrone.Core.ImportLists.ImportListMovies;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Movies;
using NzbDrone.Core.Test.Framework;

Expand Down Expand Up @@ -59,8 +60,7 @@ public void Setup()
_importListFetch = new ImportListFetchResult
{
Movies = _list1Movies,
AnyFailure = false,
SyncedLists = 1
AnyFailure = false
};

_commandAll = new ImportListSyncCommand
Expand All @@ -84,13 +84,21 @@ public void Setup()
.Setup(v => v.MovieExists(It.IsAny<Movie>()))
.Returns(false);

Mocker.GetMock<IMovieService>()
.Setup(v => v.MovieExists(It.IsAny<Movie>()))
.Returns(false);

Mocker.GetMock<IMovieService>()
.Setup(v => v.AllMovieTmdbIds())
.Returns(new List<int>());

Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
.Returns(_importListFetch);

Mocker.GetMock<ISearchForNewMovie>()
.Setup(v => v.MapMovieToTmdbMovie(It.IsAny<MovieMetadata>()))
.Returns<MovieMetadata>(m => new MovieMetadata { TmdbId = m.TmdbId });
}

private void GivenListFailure()
Expand All @@ -100,7 +108,8 @@ private void GivenListFailure()

private void GivenNoListSync()
{
_importListFetch.SyncedLists = 0;
_importListFetch.SyncedLists = new List<int>();
_importListFetch.SyncedWithoutFailure = new List<int>();
}

private void GivenCleanLevel(string cleanLevel)
Expand All @@ -114,6 +123,9 @@ private void GivenList(int id, bool enabledAuto)
{
var importListDefinition = new ImportListDefinition { Id = id, EnableAuto = enabledAuto };

_importListFetch.SyncedLists.Add(id);
_importListFetch.SyncedWithoutFailure.Add(id);

Mocker.GetMock<IImportListFactory>()
.Setup(v => v.Get(id))
.Returns(importListDefinition);
Expand Down
74 changes: 18 additions & 56 deletions src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@
using NLog;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Common.TPL;
using NzbDrone.Core.ImportLists.ImportListMovies;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Movies;

namespace NzbDrone.Core.ImportLists
{
Expand All @@ -21,26 +18,14 @@ public class FetchAndParseImportListService : IFetchAndParseImportList
{
private readonly IImportListFactory _importListFactory;
private readonly IImportListStatusService _importListStatusService;
private readonly IImportListMovieService _listMovieService;
private readonly ISearchForNewMovie _movieSearch;
private readonly IProvideMovieInfo _movieInfoService;
private readonly IMovieMetadataService _movieMetadataService;
private readonly Logger _logger;

public FetchAndParseImportListService(IImportListFactory importListFactory,
IImportListStatusService importListStatusService,
IImportListMovieService listMovieService,
ISearchForNewMovie movieSearch,
IProvideMovieInfo movieInfoService,
IMovieMetadataService movieMetadataService,
Logger logger)
{
_importListFactory = importListFactory;
_importListStatusService = importListStatusService;
_listMovieService = listMovieService;
_movieSearch = movieSearch;
_movieInfoService = movieInfoService;
_movieMetadataService = movieMetadataService;
_logger = logger;
}

Expand Down Expand Up @@ -101,21 +86,17 @@ public ImportListFetchResult Fetch()

if (!importListReports.AnyFailure)
{
var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId));
var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId))).Where(x => x.TmdbId > 0).ToList();
var listMovies = importListReports.Movies;

listMovies.AddRange(alreadyMapped);
listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList();
listMovies.ForEach(m => m.ListId = importList.Definition.Id);

result.Movies.AddRange(listMovies);
_listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id);

result.SyncedWithoutFailure.Add(importList.Definition.Id);
}

result.AnyFailure |= importListReports.AnyFailure;
result.SyncedLists++;

_importListStatusService.UpdateListSyncStatus(importList.Definition.Id);
result.SyncedLists.Add(importList.Definition.Id);
}
}
catch (Exception e)
Expand All @@ -129,9 +110,17 @@ public ImportListFetchResult Fetch()

Task.WaitAll(taskList.ToArray());

foreach (var list in importLists)
{
if (result.SyncedLists.Contains(list.Definition.Id))
{
_importListStatusService.UpdateListSyncStatus(list.Definition.Id);
}
}

result.Movies = result.Movies.DistinctBy(r => new { r.TmdbId, r.ImdbId, r.Title }).ToList();

_logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists);
_logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists.Count);

return result;
}
Expand Down Expand Up @@ -160,19 +149,19 @@ public ImportListFetchResult FetchSingleList(ImportListDefinition definition)

if (!importListReports.AnyFailure)
{
var listMovies = MapMovieReports(importListReports.Movies)
.Where(x => x.TmdbId > 0)
.DistinctBy(x => x.TmdbId)
.ToList();
var listMovies = importListReports.Movies;

listMovies.ForEach(m => m.ListId = importList.Definition.Id);

result.Movies.AddRange(listMovies);
_listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id);

result.SyncedWithoutFailure.Add(importList.Definition.Id);
}

result.AnyFailure |= importListReports.AnyFailure;

result.SyncedLists.Add(importList.Definition.Id);

_importListStatusService.UpdateListSyncStatus(importList.Definition.Id);
}
}
Expand All @@ -187,32 +176,5 @@ public ImportListFetchResult FetchSingleList(ImportListDefinition definition)

return result;
}

private List<ImportListMovie> MapMovieReports(IEnumerable<ImportListMovie> reports)
{
var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year }))
.Where(x => x != null)
.DistinctBy(x => x.TmdbId)
.ToList();

_movieMetadataService.UpsertMany(mappedMovies);

var mappedListMovies = new List<ImportListMovie>();

foreach (var movieMeta in mappedMovies)
{
var mappedListMovie = new ImportListMovie();

if (movieMeta != null)
{
mappedListMovie.MovieMetadata = movieMeta;
mappedListMovie.MovieMetadataId = movieMeta.Id;
}

mappedListMovies.Add(mappedListMovie);
}

return mappedListMovies;
}
}
}
5 changes: 4 additions & 1 deletion src/NzbDrone.Core/ImportLists/ImportListBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ public class ImportListFetchResult
public ImportListFetchResult()
{
Movies = new List<ImportListMovie>();
SyncedLists = new List<int>();
SyncedWithoutFailure = new List<int>();
}

public List<ImportListMovie> Movies { get; set; }
public bool AnyFailure { get; set; }
public int SyncedLists { get; set; }
public List<int> SyncedLists { get; set; }
public List<int> SyncedWithoutFailure { get; set; }
}

public abstract class ImportListBase<TSettings> : IImportList
Expand Down
62 changes: 57 additions & 5 deletions src/NzbDrone.Core/ImportLists/ImportListSyncService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using NzbDrone.Core.ImportLists.ImportExclusions;
using NzbDrone.Core.ImportLists.ImportListMovies;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.Movies;

namespace NzbDrone.Core.ImportLists
Expand All @@ -17,6 +18,8 @@ public class ImportListSyncService : IExecute<ImportListSyncCommand>
private readonly IImportListFactory _importListFactory;
private readonly IFetchAndParseImportList _listFetcherAndParser;
private readonly IMovieService _movieService;
private readonly IMovieMetadataService _movieMetadataService;
private readonly ISearchForNewMovie _movieSearch;
private readonly IAddMovieService _addMovieService;
private readonly IConfigService _configService;
private readonly IImportExclusionsService _exclusionService;
Expand All @@ -25,6 +28,8 @@ public class ImportListSyncService : IExecute<ImportListSyncCommand>
public ImportListSyncService(IImportListFactory importListFactory,
IFetchAndParseImportList listFetcherAndParser,
IMovieService movieService,
IMovieMetadataService movieMetadataService,
ISearchForNewMovie movieSearch,
IAddMovieService addMovieService,
IConfigService configService,
IImportExclusionsService exclusionService,
Expand All @@ -34,6 +39,8 @@ public class ImportListSyncService : IExecute<ImportListSyncCommand>
_importListFactory = importListFactory;
_listFetcherAndParser = listFetcherAndParser;
_movieService = movieService;
_movieMetadataService = movieMetadataService;
_movieSearch = movieSearch;
_addMovieService = addMovieService;
_exclusionService = exclusionService;
_listMovieService = listMovieService;
Expand All @@ -52,17 +59,17 @@ private void SyncAll()

var listItemsResult = _listFetcherAndParser.Fetch();

if (listItemsResult.SyncedLists == 0)
if (listItemsResult.SyncedLists.Count == 0)
{
return;
}

ProcessListItems(listItemsResult);

if (!listItemsResult.AnyFailure)
{
CleanLibrary();
}

ProcessListItems(listItemsResult);
}

private void SyncList(ImportListDefinition definition)
Expand Down Expand Up @@ -125,7 +132,25 @@ private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie

private void ProcessListItems(ImportListFetchResult listFetchResult)
{
listFetchResult.Movies = listFetchResult.Movies.DistinctBy(x =>
var allMappedMovies = new List<ImportListMovie>();

// Sync ListMovies table for Discovery view and Cleaning task
foreach (var listId in listFetchResult.SyncedWithoutFailure)
{
var listMovies = listFetchResult.Movies.Where(x => x.ListId == listId);
var alreadyMapped = allMappedMovies.Where(x => listMovies.Any(r => r.TmdbId == x.TmdbId));
var mappedListMovies = MapMovieReports(listMovies.Where(x => allMappedMovies.All(r => r.TmdbId != x.TmdbId)).ToList()).Where(x => x.TmdbId > 0).ToList();

mappedListMovies.AddRange(alreadyMapped);
mappedListMovies = mappedListMovies.DistinctBy(x => x.TmdbId).ToList();
mappedListMovies.ForEach(m => m.ListId = listId);

allMappedMovies.AddRange(mappedListMovies);

_listMovieService.SyncMoviesForList(mappedListMovies, listId);
}

allMappedMovies = allMappedMovies.DistinctBy(x =>
{
if (x.TmdbId != 0)
{
Expand All @@ -140,7 +165,7 @@ private void ProcessListItems(ImportListFetchResult listFetchResult)
return x.Title;
}).ToList();

var listedMovies = listFetchResult.Movies.ToList();
var listedMovies = allMappedMovies;

var importExclusions = _exclusionService.GetAllExclusions();
var dbMovies = _movieService.AllMovieTmdbIds();
Expand Down Expand Up @@ -168,6 +193,33 @@ private void ProcessListItems(ImportListFetchResult listFetchResult)
}
}

private List<ImportListMovie> MapMovieReports(IEnumerable<ImportListMovie> reports)
{
var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year }))
.Where(x => x != null)
.DistinctBy(x => x.TmdbId)
.ToList();

_movieMetadataService.UpsertMany(mappedMovies);

var mappedListMovies = new List<ImportListMovie>();

foreach (var movieMeta in mappedMovies)
{
var mappedListMovie = new ImportListMovie();

if (movieMeta != null)
{
mappedListMovie.MovieMetadata = movieMeta;
mappedListMovie.MovieMetadataId = movieMeta.Id;
}

mappedListMovies.Add(mappedListMovie);
}

return mappedListMovies;
}

public void Execute(ImportListSyncCommand message)
{
if (message.DefinitionId.HasValue)
Expand Down
Loading