Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Metadata syncing rework #349

Merged
merged 9 commits into from Aug 3, 2020
Merged

Metadata syncing rework #349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5f34222
Optimize network calls for music syncing
mcarlton00 Aug 2, 2020
2c55f7a
Remove library_check wrapper from songs
mcarlton00 Aug 2, 2020
0ee5b61
stupid getitems
mcarlton00 Aug 2, 2020
eca9630
Remove library_check wrapper
mcarlton00 Aug 2, 2020
4178f0a
Fix boxset syncing
mcarlton00 Aug 2, 2020
7f012dc
Fix undefined variable in remove function
mcarlton00 Aug 2, 2020
dc6c9e7
Fix code smells for unused variables
mcarlton00 Aug 2, 2020
ce21b37
Remove unneeded message variable
mcarlton00 Aug 3, 2020
d10ca84
empty default if there are no boxsets in the server
mcarlton00 Aug 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
49 changes: 29 additions & 20 deletions jellyfin_kodi/downloader.py
View file
Open in desktop
Expand Up @@ -189,31 +189,40 @@ def get_items(parent_id, item_type=None, basic=False, params=None):
yield items


def get_artists(parent_id=None, basic=False, params=None, server_id=None):
def get_artists(parent_id=None):

query = {
'url': "Artists",
'params': {
'UserId': "{UserId}",
'ParentId': parent_id,
'SortBy': "SortName",
'SortOrder': "Ascending",
'Fields': api.basic_info() if basic else api.music_info(),
'CollapseBoxSetItems': False,
'IsVirtualUnaired': False,
'EnableTotalRecordCount': False,
'LocationTypes': "FileSystem,Remote,Offline",
'IsMissing': False,
'Recursive': True
}
url = "Artists"

params = {
'UserId': "{UserId}",
'ParentId': parent_id,
'SortBy': "SortName",
'SortOrder': "Ascending",
'Fields': api.music_info(),
'CollapseBoxSetItems': False,
'IsVirtualUnaired': False,
'EnableTotalRecordCount': False,
'LocationTypes': "FileSystem,Remote,Offline",
'IsMissing': False,
'Recursive': True
}

if params:
query['params'].update(params)
return _get(url, params)

for items in _get_items(query, server_id):
yield items

def get_library_items(library_id, item_type):
url = "Users/{UserId}/Items"

params = {
'ParentId': library_id,
'IncludeItemTypes': item_type,
'SortBy': "SortName",
'SortOrder': "Ascending",
'Fields': api.info(),
'Recursive': True,
}

return _get(url, params)

def get_albums_by_artist(artist_id, basic=False):

Expand Down
115 changes: 63 additions & 52 deletions jellyfin_kodi/full_sync.py
View file
Open in desktop
Expand Up @@ -213,11 +213,19 @@ def process_library(self, library_id):
}
try:
if library_id.startswith('Boxsets:'):
boxset_library = {}

if library_id.endswith('Refresh'):
self.refresh_boxsets()
else:
self.boxsets(library_id.split('Boxsets:')[1] if len(library_id) > len('Boxsets:') else None)
libraries = self.get_libraries(library_id.split('Boxsets:')[1] if len(library_id) > len('Boxsets:') else None)
for entry in libraries:
if entry[2] == 'boxsets':
boxset_library = {'Id': entry[0], 'Name': entry[1]}
break

if boxset_library:
if library_id.endswith('Refresh'):
self.refresh_boxsets(boxset_library)
else:
self.boxsets(boxset_library)

return

Expand Down Expand Up @@ -270,7 +278,7 @@ def movies(self, library, dialog):
for items in server.get_items(library['Id'], "Movie", False, self.sync['RestorePoint'].get('params')):

with self.video_database_locks() as (videodb, jellyfindb):
obj = Movies(self.server, jellyfindb, videodb, self.direct_path)
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)

self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
Expand All @@ -280,11 +288,11 @@ def movies(self, library, dialog):
dialog.update(int((float(start_index + index) / float(items['TotalRecordCount'])) * 100),
heading="%s: %s" % (translate('addon_name'), library['Name']),
message=movie['Name'])
obj.movie(movie, library=library)
obj.movie(movie)
processed_ids.append(movie['Id'])

with self.video_database_locks() as (videodb, jellyfindb):
obj = Movies(self.server, jellyfindb, videodb, self.direct_path)
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
obj.item_ids = processed_ids

if self.update_library:
Expand Down Expand Up @@ -313,7 +321,7 @@ def tvshows(self, library, dialog):
for items in server.get_items(library['Id'], "Series", False, self.sync['RestorePoint'].get('params')):

with self.video_database_locks() as (videodb, jellyfindb):
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, True)
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, library, True)

self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
Expand All @@ -324,7 +332,7 @@ def tvshows(self, library, dialog):
message = show['Name']
dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=message)

if obj.tvshow(show, library=library) is not False:
if obj.tvshow(show) is not False:

for episodes in server.get_episode_by_show(show['Id']):
for episode in episodes['Items']:
Expand All @@ -334,7 +342,7 @@ def tvshows(self, library, dialog):
processed_ids.append(show['Id'])

with self.video_database_locks() as (videodb, jellyfindb):
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, True)
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, library, True)
obj.item_ids = processed_ids
if self.update_library:
self.tvshows_compare(library, obj, jellyfindb)
Expand Down Expand Up @@ -365,7 +373,7 @@ def musicvideos(self, library, dialog):
for items in server.get_items(library['Id'], "MusicVideo", False, self.sync['RestorePoint'].get('params')):

with self.video_database_locks() as (videodb, jellyfindb):
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path)
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path, library)

self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
Expand All @@ -375,11 +383,11 @@ def musicvideos(self, library, dialog):
dialog.update(int((float(start_index + index) / float(items['TotalRecordCount'])) * 100),
heading="%s: %s" % (translate('addon_name'), library['Name']),
message=mvideo['Name'])
obj.musicvideo(mvideo, library=library)
obj.musicvideo(mvideo)
processed_ids.append(mvideo['Id'])

with self.video_database_locks() as (videodb, jellyfindb):
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path)
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path, library)
obj.item_ids = processed_ids
if self.update_library:
self.musicvideos_compare(library, obj, jellyfindb)
Expand All @@ -405,38 +413,41 @@ def music(self, library, dialog):
with self.library.music_database_lock:
with Database('music') as musicdb:
with Database('jellyfin') as jellyfindb:
obj = Music(self.server, jellyfindb, musicdb, self.direct_path)

for items in server.get_artists(library['Id'], False, self.sync['RestorePoint'].get('params')):

self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']

for index, artist in enumerate(items['Items']):

percent = int((float(start_index + index) / float(items['TotalRecordCount'])) * 100)
message = artist['Name']
dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=message)
obj.artist(artist, library=library)

for albums in server.get_albums_by_artist(artist['Id']):

for album in albums['Items']:
obj.album(album)

for songs in server.get_items(album['Id'], "Audio"):
for song in songs['Items']:

dialog.update(percent,
message="%s/%s/%s" % (message, album['Name'][:7], song['Name'][:7]))
obj.song(song)

for songs in server.get_songs_by_artist(artist['Id']):
for song in songs['Items']:

dialog.update(percent, message="%s/%s" % (message, song['Name']))
obj.song(song)

obj = Music(self.server, jellyfindb, musicdb, self.direct_path, library)

library_id = library['Id']

# Get all items in the library to process locally
artists_data = server.get_artists(library_id)
artists = artists_data['Items']
num_artists = artists_data['TotalRecordCount']
albums = server.get_library_items(library_id, 'MusicAlbum')['Items']
songs = server.get_library_items(library_id, 'Audio')['Items']

for index, artist in enumerate(artists):
artist_name = artist.get('Name')

# Update percentage dialog
percent = int((float(index) / float(num_artists)) * 100)
dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=artist_name)

# Add artist to database
obj.artist(artist)

# Get all albums for each artist
artist_albums = [ album for album in albums if artist_name in album.get('Artists') ]
for album in artist_albums:
# Add album to database
obj.album(album)
album_id = album.get('Id')
# Get all songs in each album
album_songs = [ song for song in songs if album_id == song.get('AlbumId') ]
for song in album_songs:
dialog.update(percent,
message="%s/%s/%s" % (artist_name, album['Name'][:7], song['Name'][:7]))
# Add song to database
obj.song(song)
#
if self.update_library:
self.music_compare(library, obj, jellyfindb)

Expand All @@ -457,14 +468,14 @@ def music_compare(self, library, obj, jellyfindb):
obj.remove(x[0])

@progress(translate(33018))
def boxsets(self, library_id=None, dialog=None):
def boxsets(self, library, dialog=None):

''' Process all boxsets.
'''
for items in server.get_items(library_id, "BoxSet", False, self.sync['RestorePoint'].get('params')):
for items in server.get_items(library['Id'], "BoxSet", False, self.sync['RestorePoint'].get('params')):

with self.video_database_locks() as (videodb, jellyfindb):
obj = Movies(self.server, jellyfindb, videodb, self.direct_path)
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)

self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
Expand All @@ -476,12 +487,12 @@ def boxsets(self, library_id=None, dialog=None):
message=boxset['Name'])
obj.boxset(boxset)

def refresh_boxsets(self):
def refresh_boxsets(self, library):

''' Delete all exisitng boxsets and re-add.
'''
with self.video_database_locks() as (videodb, jellyfindb):
obj = Movies(self.server, jellyfindb, videodb, self.direct_path)
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
obj.boxsets_reset()

self.boxsets(None)
Expand Down Expand Up @@ -514,15 +525,15 @@ def remove_library(self, library_id, dialog):
movies = [x for x in items if x[1] == 'Movie']
tvshows = [x for x in items if x[1] == 'Series']

obj = Movies(self.server, jellyfindb, kodidb, direct_path).remove
obj = Movies(self.server, jellyfindb, kodidb, direct_path, library).remove

for item in movies:

obj(item[0])
dialog.update(int((float(count) / float(len(items)) * 100)), heading="%s: %s" % (translate('addon_name'), library[0]))
count += 1

obj = TVShows(self.server, jellyfindb, kodidb, direct_path).remove
obj = TVShows(self.server, jellyfindb, kodidb, direct_path, library).remove

for item in tvshows:

Expand Down
1 change: 0 additions & 1 deletion jellyfin_kodi/helper/__init__.py
View file
Open in desktop
Expand Up @@ -27,4 +27,3 @@
from .wrapper import progress
from .wrapper import stop
from .wrapper import jellyfin_item
from .wrapper import library_check
16 changes: 16 additions & 0 deletions jellyfin_kodi/helper/utils.py
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@
from . import LazyLogger
from .translate import translate


#################################################################################################

LOG = LazyLogger(__name__)
Expand Down Expand Up @@ -535,3 +536,18 @@ def get_filesystem_encoding():
enc = 'utf-8'

return enc


def find_library(server, item):
from database import get_sync
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to move import to top of file instead of in function

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't work. No, I don't know why, but it throws import errors in the database init. Mystery for another day, imo.


sync = get_sync()

ancestors = server.jellyfin.get_ancestors(item['Id'])
for ancestor in ancestors:
if ancestor['Id'] in sync['Whitelist']:
LOG.info('Ancestor Found')
return ancestor

LOG.error('No ancestor found, not syncing item with ID: {}'.format(item['Id']))
return {}
57 changes: 0 additions & 57 deletions jellyfin_kodi/helper/wrapper.py
View file
Open in desktop
Expand Up @@ -77,60 +77,3 @@ def wrapper(self, item, *args, **kwargs):
return func(self, item, e_item=e_item, *args, **kwargs)

return wrapper


def library_check(func):

''' Wrapper to retrieve the library
'''
def wrapper(self, item, *args, **kwargs):

''' TODO: Rethink this one... songs and albums cannot be found by library. expensive.
'''
from database import get_sync

if kwargs.get('library') is None:
sync = get_sync()

if 'e_item' in kwargs:
try:
view_id = kwargs['e_item'][6]
view_name = self.jellyfin_db.get_view_name(view_id)
view = {'Name': view_name, 'Id': view_id}
except Exception:
view = None

if view is None:
ancestors = self.server.jellyfin.get_ancestors(item['Id'])

if not ancestors:
if item['Type'] == 'MusicArtist':

try:
views = self.jellyfin_db.get_views_by_media('music')[0]
except Exception as error:
LOG.exception(error)
return

view = {'Id': views[0], 'Name': views[1]}
else: # Grab the first music library
return
else:
for ancestor in ancestors:
if ancestor['Type'] == 'CollectionFolder':

view = self.jellyfin_db.get_view_name(ancestor['Id'])
view = {'Id': None, 'Name': None} if view is None else {'Name': ancestor['Name'], 'Id': ancestor['Id']}

break

if view['Id'] not in [x.replace('Mixed:', "") for x in sync['Whitelist'] + sync['Libraries']]:
LOG.info("Library %s is not synced. Skip update.", view['Id'])

return

kwargs['library'] = view

return func(self, item, *args, **kwargs)

return wrapper