Skip to content
Newer
Older
100644 812 lines (704 sloc) 26 KB
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
1 # This file is part of beets.
2 # Copyright 2011, Adrian Sampson.
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining
5 # a copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish,
8 # distribute, sublicense, and/or sell copies of the Software, and to
9 # permit persons to whom the Software is furnished to do so, subject to
10 # the following conditions:
11 #
12 # The above copyright notice and this permission notice shall be
13 # included in all copies or substantial portions of the Software.
14
15 """Provides the basic, interface-agnostic workflow for importing and
16 autotagging music files.
17 """
18 from __future__ import with_statement # Python 2.5
19 import os
20 import logging
21 import pickle
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
22 from collections import defaultdict
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
23
24 from beets import autotag
25 from beets import library
26 import beets.autotag.art
27 from beets import plugins
2746f1f @sampsyo move pipeline into new beets.util package
sampsyo authored
28 from beets.util import pipeline
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
29 from beets.util import syspath, normpath, plurality
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
30 from beets.util.enumeration import enum
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
31
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
32 action = enum(
a074db7 @sampsyo manual specification of MBIDs
sampsyo authored
33 'SKIP', 'ASIS', 'TRACKS', 'MANUAL', 'APPLY', 'MANUAL_ID',
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
34 name='action'
35 )
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
36
37 QUEUE_SIZE = 128
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
38 STATE_FILE = os.path.expanduser('~/.beetsstate')
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
39 SINGLE_ARTIST_THRESH = 0.25
40 VARIOUS_ARTISTS = u'Various Artists'
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
41
42 # Global logger.
43 log = logging.getLogger('beets')
44
45 class ImportAbort(Exception):
46 """Raised when the user aborts the tagging operation.
47 """
48 pass
49
50
51 # Utilities.
52
53 def tag_log(logfile, status, path):
54 """Log a message about a given album to logfile. The status should
55 reflect the reason the album couldn't be tagged.
56 """
57 if logfile:
58 print >>logfile, '%s %s' % (status, path)
59
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
60 def log_choice(config, task):
61 """Logs the task's current choice if it should be logged.
62 """
63 path = task.path if task.is_album else task.item.path
64 if task.choice_flag is action.ASIS:
65 tag_log(config.logfile, 'asis', path)
66 elif task.choice_flag is action.SKIP:
67 tag_log(config.logfile, 'skip', path)
68
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
69 def _reopen_lib(lib):
70 """Because of limitations in SQLite, a given Library is bound to
71 the thread in which it was created. This function reopens Library
72 objects so that they can be used from separate threads.
73 """
74 if isinstance(lib, library.Library):
75 return library.Library(
76 lib.path,
77 lib.directory,
78 lib.path_formats,
79 lib.art_filename,
80 )
81 else:
82 return lib
83
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
84 def _duplicate_check(lib, task, recent=None):
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
85 """Check whether an album already exists in the library. `recent`
86 should be a set of (artist, album) pairs that will be built up
87 with every call to this function and checked along with the
88 library.
89 """
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
90 if task.choice_flag is action.ASIS:
91 artist = task.cur_artist
92 album = task.cur_album
93 elif task.choice_flag is action.APPLY:
95f38db @sampsyo "info dictionaries" replaced with AlbumInfo and TrackInfo
sampsyo authored
94 artist = task.info.artist
95 album = task.info.album
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
96 else:
97 return False
98
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
99 if artist is None:
100 # As-is import with no artist. Skip check.
101 return False
102
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
103 # Try the recent albums.
104 if recent is not None:
105 if (artist, album) in recent:
106 return True
107 recent.add((artist, album))
108
109 # Look in the library.
9a7a551 @laarmen Enable import of incomplete albums
laarmen authored
110 cur_paths = set(i.path for i in task.items if i)
8341dee @sampsyo reorder items() and albums() parameters to reflect common use
sampsyo authored
111 for album_cand in lib.albums(artist=artist):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
112 if album_cand.album == album:
f313015 @sampsyo don't count existing items/albums as duplicates (allowing update)
sampsyo authored
113 # Check whether the album is identical in contents, in which
114 # case it is not a duplicate (will be replaced).
115 other_paths = set(i.path for i in album_cand.items())
116 if other_paths == cur_paths:
117 continue
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
118 return True
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
119
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
120 return False
121
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
122 def _item_duplicate_check(lib, task, recent=None):
7e89282 @sampsyo duplicate detection for items
sampsyo authored
123 """Check whether an item already exists in the library."""
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
124 if task.choice_flag is action.ASIS:
125 artist = task.item.artist
126 title = task.item.title
127 elif task.choice_flag is action.APPLY:
95f38db @sampsyo "info dictionaries" replaced with AlbumInfo and TrackInfo
sampsyo authored
128 artist = task.info.artist
129 title = task.info.title
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
130 else:
131 return False
132
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
133 # Try recent items.
134 if recent is not None:
135 if (artist, title) in recent:
136 return True
137 recent.add((artist, title))
138
139 # Check the library.
140 item_iter = lib.items(artist=artist, title=title)
7e89282 @sampsyo duplicate detection for items
sampsyo authored
141 try:
f313015 @sampsyo don't count existing items/albums as duplicates (allowing update)
sampsyo authored
142 for other_item in item_iter:
143 # Existing items not considered duplicates.
144 if other_item.path == task.item.path:
145 continue
146 return True
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
147 finally:
148 item_iter.close()
f313015 @sampsyo don't count existing items/albums as duplicates (allowing update)
sampsyo authored
149 return False
7e89282 @sampsyo duplicate detection for items
sampsyo authored
150
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
151 def _infer_album_fields(task):
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
152 """Given an album and an associated import task, massage the
153 album-level metadata. This ensures that the album artist is set
154 and that the "compilation" flag is set automatically.
155 """
156 assert task.is_album
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
157 assert task.items
158
159 changes = {}
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
160
161 if task.choice_flag == action.ASIS:
162 # Taking metadata "as-is". Guess whether this album is VA.
163 plur_artist, freq = plurality([i.artist for i in task.items])
951e4ee @sampsyo fix VA inference for small (1-track) albums
sampsyo authored
164 if freq == len(task.items) or (freq > 1 and
165 float(freq) / len(task.items) >= SINGLE_ARTIST_THRESH):
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
166 # Single-artist album.
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
167 changes['albumartist'] = plur_artist
168 changes['comp'] = False
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
169 else:
170 # VA.
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
171 changes['albumartist'] = VARIOUS_ARTISTS
172 changes['comp'] = True
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
173
174 elif task.choice_flag == action.APPLY:
175 # Applying autotagged metadata. Just get AA from the first
176 # item.
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
177 if not task.items[0].albumartist:
178 changes['albumartist'] = task.items[0].artist
179 if not task.items[0].mb_albumartistid:
180 changes['mb_albumartistid'] = task.items[0].mb_artistid
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
181
182 else:
183 assert False
184
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
185 # Apply new metadata.
186 for item in task.items:
187 for k, v in changes.iteritems():
188 setattr(item, k, v)
189
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
190 def _open_state():
191 """Reads the state file, returning a dictionary."""
192 try:
193 with open(STATE_FILE) as f:
194 return pickle.load(f)
195 except IOError:
196 return {}
197 def _save_state(state):
198 """Writes the state dictionary out to disk."""
199 with open(STATE_FILE, 'w') as f:
200 pickle.dump(state, f)
201
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
202
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
203 # Utilities for reading and writing the beets progress file, which
204 # allows long tagging tasks to be resumed when they pause (or crash).
205 PROGRESS_KEY = 'tagprogress'
206 def progress_set(toppath, path):
207 """Record that tagging for the given `toppath` was successful up to
208 `path`. If path is None, then clear the progress value (indicating
209 that the tagging completed).
210 """
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
211 state = _open_state()
212 if PROGRESS_KEY not in state:
213 state[PROGRESS_KEY] = {}
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
214
215 if path is None:
216 # Remove progress from file.
217 if toppath in state[PROGRESS_KEY]:
218 del state[PROGRESS_KEY][toppath]
219 else:
220 state[PROGRESS_KEY][toppath] = path
221
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
222 _save_state(state)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
223 def progress_get(toppath):
224 """Get the last successfully tagged subpath of toppath. If toppath
225 has no progress information, returns None.
226 """
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
227 state = _open_state()
228 if PROGRESS_KEY not in state:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
229 return None
230 return state[PROGRESS_KEY].get(toppath)
231
232
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
233 # Similarly, utilities for manipulating the "incremental" import log.
234 # This keeps track of all directories that were ever imported, which
235 # allows the importer to only import new stuff.
236 HISTORY_KEY = 'taghistory'
237 def history_add(path):
238 """Indicate that the import of `path` is completed and should not
239 be repeated in incremental imports.
240 """
241 state = _open_state()
242 if HISTORY_KEY not in state:
243 state[HISTORY_KEY] = set()
244
245 state[HISTORY_KEY].add(path)
246
247 _save_state(state)
248 def history_get():
249 """Get the set of completed paths in incremental imports.
250 """
251 state = _open_state()
252 if HISTORY_KEY not in state:
253 return set()
254 return state[HISTORY_KEY]
255
256
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
257 # The configuration structure.
258
259 class ImportConfig(object):
260 """Contains all the settings used during an import session. Should
261 be used in a "write-once" way -- everything is set up initially and
262 then never touched again.
263 """
6884fc2 @sampsyo don't use __slots__
sampsyo authored
264 _fields = ['lib', 'paths', 'resume', 'logfile', 'color', 'quiet',
265 'quiet_fallback', 'copy', 'write', 'art', 'delete',
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
266 'choose_match_func', 'should_resume_func', 'threaded',
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
267 'autot', 'singletons', 'timid', 'choose_item_func',
5965b37 @sampsyo skip (configurable) clutter filenames when importing
sampsyo authored
268 'query', 'incremental', 'ignore']
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
269 def __init__(self, **kwargs):
6884fc2 @sampsyo don't use __slots__
sampsyo authored
270 for slot in self._fields:
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
271 setattr(self, slot, kwargs[slot])
272
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
273 # Normalize the paths.
274 if self.paths:
275 self.paths = map(normpath, self.paths)
276
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
277 # Incremental and progress are mutually exclusive.
278 if self.incremental:
279 self.resume = False
280
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
281 # When based on a query instead of directories, never
282 # save progress or try to resume.
283 if self.query is not None:
284 self.paths = None
285 self.resume = False
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
286 self.incremental = False
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
287
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
288
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
289 # The importer task class.
290
291 class ImportTask(object):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
292 """Represents a single set of items to be imported along with its
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
293 intermediate state. May represent an album or a single item.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
294 """
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
295 def __init__(self, toppath=None, path=None, items=None):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
296 self.toppath = toppath
297 self.path = path
298 self.items = items
299 self.sentinel = False
300
301 @classmethod
302 def done_sentinel(cls, toppath):
303 """Create an ImportTask that indicates the end of a top-level
304 directory import.
305 """
306 obj = cls(toppath)
307 obj.sentinel = True
308 return obj
309
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
310 @classmethod
a916551 @sampsyo progress/resume for item imports
sampsyo authored
311 def progress_sentinel(cls, toppath, path):
312 """Create a task indicating that a single directory in a larger
313 import has finished. This is only required for singleton
314 imports; progress is implied for album imports.
315 """
316 obj = cls(toppath, path)
317 obj.sentinel = True
318 return obj
319
320 @classmethod
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
321 def item_task(cls, item):
322 """Creates an ImportTask for a single item."""
323 obj = cls()
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
324 obj.item = item
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
325 obj.is_album = False
326 return obj
327
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
328 def set_match(self, cur_artist, cur_album, candidates, rec):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
329 """Sets the candidates for this album matched by the
330 `autotag.tag_album` method.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
331 """
332 assert not self.sentinel
333 self.cur_artist = cur_artist
334 self.cur_album = cur_album
335 self.candidates = candidates
336 self.rec = rec
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
337 self.is_album = True
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
338
339 def set_null_match(self):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
340 """Set the candidates to indicate no album match was found.
341 """
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
342 self.set_match(None, None, None, None)
343
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
344 def set_item_match(self, candidates, rec):
345 """Set the match for a single-item task."""
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
346 assert not self.is_album
347 assert self.item is not None
348 self.item_match = (candidates, rec)
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
349
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
350 def set_null_item_match(self):
351 """For single-item tasks, mark the item as having no matches.
352 """
353 assert not self.is_album
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
354 assert self.item is not None
355 self.item_match = None
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
356
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
357 def set_choice(self, choice):
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
358 """Given either an (info, items) tuple or an action constant,
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
359 indicates that an action has been selected by the user (or
360 automatically).
361 """
362 assert not self.sentinel
a074db7 @sampsyo manual specification of MBIDs
sampsyo authored
363 # Not part of the task structure:
364 assert choice not in (action.MANUAL, action.MANUAL_ID)
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
365 assert choice != action.APPLY # Only used internally.
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
366 if choice in (action.SKIP, action.ASIS, action.TRACKS):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
367 self.choice_flag = choice
368 self.info = None
369 else:
86b1f92 @sampsyo fix some (suprisingly few) airplane-coding bugs
sampsyo authored
370 assert not isinstance(choice, action)
371 if self.is_album:
372 info, items = choice
373 self.items = items # Reordered items list.
374 else:
375 info = choice
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
376 self.info = info
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
377 self.choice_flag = action.APPLY # Implicit choice.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
378
379 def save_progress(self):
380 """Updates the progress state to indicate that this album has
381 finished.
382 """
a916551 @sampsyo progress/resume for item imports
sampsyo authored
383 if self.sentinel and self.path is None:
384 # "Done" sentinel.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
385 progress_set(self.toppath, None)
a916551 @sampsyo progress/resume for item imports
sampsyo authored
386 elif self.sentinel or self.is_album:
387 # "Directory progress" sentinel for singletons or a real
388 # album task, which implies the same.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
389 progress_set(self.toppath, self.path)
390
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
391 def save_history(self):
392 """Save the directory in the history for incremental imports.
393 """
394 if self.sentinel or self.is_album:
395 history_add(self.path)
396
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
397 # Logical decisions.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
398 def should_write_tags(self):
399 """Should new info be written to the files' metadata?"""
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
400 if self.choice_flag == action.APPLY:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
401 return True
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
402 elif self.choice_flag in (action.ASIS, action.TRACKS, action.SKIP):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
403 return False
404 else:
405 assert False
406 def should_fetch_art(self):
407 """Should album art be downloaded for this album?"""
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
408 return self.should_write_tags() and self.is_album
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
409 def should_skip(self):
410 """After a choice has been made, returns True if this is a
411 sentinel or it has been marked for skipping.
412 """
413 return self.sentinel or self.choice_flag == action.SKIP
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
414
415
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
416 # Full-album pipeline stages.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
417
a916551 @sampsyo progress/resume for item imports
sampsyo authored
418 def read_tasks(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
419 """A generator yielding all the albums (as ImportTask objects) found
3b23198 @sampsyo singleton imports can take single-file arguments (#184)
sampsyo authored
420 in the user-specified list of paths. In the case of a singleton
421 import, yields single-item tasks instead.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
422 """
423 # Look for saved progress.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
424 progress = config.resume is not False
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
425 if progress:
426 resume_dirs = {}
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
427 for path in config.paths:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
428 resume_dir = progress_get(path)
429 if resume_dir:
430
431 # Either accept immediately or prompt for input to decide.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
432 if config.resume:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
433 do_resume = True
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
434 log.warn('Resuming interrupted import of %s' % path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
435 else:
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
436 do_resume = config.should_resume_func(config, path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
437
438 if do_resume:
439 resume_dirs[path] = resume_dir
440 else:
441 # Clear progress; we're starting from the top.
442 progress_set(path, None)
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
443
444 # Look for saved incremental directories.
445 if config.incremental:
446 history_dirs = history_get()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
447
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
448 for toppath in config.paths:
3b23198 @sampsyo singleton imports can take single-file arguments (#184)
sampsyo authored
449 # Check whether the path is to a file.
450 if config.singletons and not os.path.isdir(syspath(toppath)):
451 item = library.Item.from_path(toppath)
452 yield ImportTask.item_task(item)
453 continue
454
455 # Produce paths under this directory.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
456 if progress:
457 resume_dir = resume_dirs.get(toppath)
5965b37 @sampsyo skip (configurable) clutter filenames when importing
sampsyo authored
458 for path, items in autotag.albums_in_dir(toppath, config.ignore):
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
459 # Skip according to progress.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
460 if progress and resume_dir:
461 # We're fast-forwarding to resume a previous tagging.
462 if path == resume_dir:
463 # We've hit the last good path! Turn off the
464 # fast-forwarding.
465 resume_dir = None
466 continue
467
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
468 # When incremental, skip paths in the history.
469 if config.incremental and path in history_dirs:
470 continue
471
a916551 @sampsyo progress/resume for item imports
sampsyo authored
472 # Yield all the necessary tasks.
473 if config.singletons:
474 for item in items:
475 yield ImportTask.item_task(item)
476 yield ImportTask.progress_sentinel(toppath, path)
477 else:
478 yield ImportTask(toppath, path, items)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
479
480 # Indicate the directory is finished.
481 yield ImportTask.done_sentinel(toppath)
482
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
483 def query_tasks(config):
484 """A generator that works as a drop-in-replacement for read_tasks.
485 Instead of finding files from the filesystem, a query is used to
486 match items from the library.
487 """
488 lib = _reopen_lib(config.lib)
489
490 if config.singletons:
491 # Search for items.
492 items = list(lib.items(config.query))
493 for item in items:
494 yield ImportTask.item_task(item)
495
496 else:
497 # Search for albums.
498 albums = lib.albums(config.query)
499 for album in albums:
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
500 log.debug('yielding album %i: %s - %s' %
501 (album.id, album.albumartist, album.album))
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
502 items = list(album.items())
503 yield ImportTask(None, album.item_dir(), items)
504
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
505 def initial_lookup(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
506 """A coroutine for performing the initial MusicBrainz lookup for an
507 album. It accepts lists of Items and yields
508 (items, cur_artist, cur_album, candidates, rec) tuples. If no match
509 is found, all of the yielded parameters (except items) are None.
510 """
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
511 task = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
512 while True:
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
513 task = yield task
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
514 if task.sentinel:
515 continue
516
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
517 log.debug('Looking up: %s' % task.path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
518 try:
704a76b @sampsyo rename interactive_autotag to timid
sampsyo authored
519 task.set_match(*autotag.tag_album(task.items, config.timid))
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
520 except autotag.AutotagError:
521 task.set_null_match()
522
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
523 def user_query(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
524 """A coroutine for interfacing with the user about the tagging
525 process. lib is the Library to import into and logfile may be
526 a file-like object for logging the import process. The coroutine
527 accepts and yields ImportTask objects.
528 """
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
529 lib = _reopen_lib(config.lib)
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
530 recent = set()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
531 task = None
532 while True:
533 task = yield task
534 if task.sentinel:
535 continue
536
537 # Ask the user for a choice.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
538 choice = config.choose_match_func(task, config)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
539 task.set_choice(choice)
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
540 log_choice(config, task)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
541
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
542 # As-tracks: transition to singleton workflow.
543 if choice is action.TRACKS:
544 # Set up a little pipeline for dealing with the singletons.
545 item_tasks = []
1b3e7d9 @sampsyo fix singleton invocation of pipeline
sampsyo authored
546 def emitter():
547 for item in task.items:
548 yield ImportTask.item_task(item)
a916551 @sampsyo progress/resume for item imports
sampsyo authored
549 yield ImportTask.progress_sentinel(task.toppath, task.path)
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
550 def collector():
551 while True:
552 item_task = yield
553 item_tasks.append(item_task)
1b3e7d9 @sampsyo fix singleton invocation of pipeline
sampsyo authored
554 ipl = pipeline.Pipeline((emitter(), item_lookup(config),
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
555 item_query(config), collector()))
556 ipl.run_sequential()
557 task = pipeline.multiple(item_tasks)
c891dac @sampsyo fix crash with "as Tracks" import option (#244)
sampsyo authored
558 continue
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
559
560 # Check for duplicates if we have a match (or ASIS).
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
561 if _duplicate_check(lib, task, recent):
562 tag_log(config.logfile, 'duplicate', task.path)
563 log.warn("This album is already in the library!")
564 task.set_choice(action.SKIP)
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
565
566 def show_progress(config):
567 """This stage replaces the initial_lookup and user_query stages
568 when the importer is run without autotagging. It displays the album
569 name and artist as the files are added.
570 """
571 task = None
572 while True:
573 task = yield task
574 if task.sentinel:
575 continue
576
577 log.info(task.path)
578
579 # Behave as if ASIS were selected.
580 task.set_null_match()
581 task.set_choice(action.ASIS)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
582
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
583 def apply_choices(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
584 """A coroutine for applying changes to albums during the autotag
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
585 process.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
586 """
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
587 lib = _reopen_lib(config.lib)
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
588 task = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
589 while True:
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
590 task = yield task
591 if task.should_skip():
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
592 continue
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
593
9a7a551 @laarmen Enable import of incomplete albums
laarmen authored
594 items = [i for i in task.items if i] if task.is_album else [task.item]
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
595 # Clear IDs in case the items are being re-tagged.
596 for item in items:
597 item.id = None
2e066b8 @sampsyo moving when re-importing reassociates albums before renaming
sampsyo authored
598 item.album_id = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
599
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
600 # Change metadata.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
601 if task.should_write_tags():
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
602 if task.is_album:
603 autotag.apply_metadata(task.items, task.info)
604 else:
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
605 autotag.apply_item_metadata(task.item, task.info)
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
606
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
607 # Infer album-level fields.
608 if task.is_album:
609 _infer_album_fields(task)
610
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
611 # Find existing item entries that these are replacing. Old
612 # album structures are automatically cleaned up when the
613 # last item is removed.
614 replaced_items = defaultdict(list)
615 for item in items:
616 dup_items = list(lib.items(
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
617 library.MatchQuery('path', item.path)
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
618 ))
619 for dup_item in dup_items:
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
620 replaced_items[item].append(dup_item)
621 log.debug('replacing item %i: %s' % (dup_item.id, item.path))
622 log.debug('%i of %i items replaced' % (len(replaced_items),
623 len(items)))
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
624
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
625 # Move/copy files.
3e75d26 @sampsyo correctly detect item existence when copying
sampsyo authored
626 task.old_paths = [item.path for item in items]
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
627 for item in items:
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
628 if config.copy:
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
629 # If we're replacing an item, then move rather than
630 # copying.
631 do_copy = not bool(replaced_items[item])
e8b8cb1 @sampsyo refactor: move() is a method on Library (not Item)
sampsyo authored
632 lib.move(item, do_copy, task.is_album)
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
633 if config.write and task.should_write_tags():
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
634 item.write()
635
636 # Add items to library. We consolidate this at the end to avoid
637 # locking while we do the copying and tag updates.
b48ee61 @sampsyo move database commits to "finally" blocks
sampsyo authored
638 try:
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
639 # Remove old items.
640 for replaced in replaced_items.itervalues():
641 for item in replaced:
642 lib.remove(item)
643
644 # Add new ones.
b48ee61 @sampsyo move database commits to "finally" blocks
sampsyo authored
645 if task.is_album:
646 # Add an album.
9a7a551 @laarmen Enable import of incomplete albums
laarmen authored
647 album = lib.add_album([i for i in task.items if i])
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
648 task.album_id = album.id
b48ee61 @sampsyo move database commits to "finally" blocks
sampsyo authored
649 else:
650 # Add tracks.
651 for item in items:
652 lib.add(item)
653 finally:
654 lib.save()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
655
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
656 def fetch_art(config):
657 """A coroutine that fetches and applies album art for albums where
658 appropriate.
659 """
660 lib = _reopen_lib(config.lib)
661 task = None
662 while True:
663 task = yield task
664 if task.should_skip():
665 continue
666
667 if task.should_fetch_art():
e64e20c @sampsyo copy album art from filesystem based on filename heuristics (#72)
sampsyo authored
668 artpath = beets.autotag.art.art_for_album(task.info, task.path)
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
669
670 # Save the art if any was found.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
671 if artpath:
b48ee61 @sampsyo move database commits to "finally" blocks
sampsyo authored
672 try:
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
673 album = lib.get_album(task.album_id)
674 album.set_art(artpath)
b48ee61 @sampsyo move database commits to "finally" blocks
sampsyo authored
675 finally:
6291a71 @sampsyo fix "mpdupdate runs twice" bug
sampsyo authored
676 lib.save(False)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
677
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
678 def finalize(config):
679 """A coroutine that finishes up importer tasks. In particular, the
680 coroutine sends plugin events, deletes old files, and saves
681 progress. This is a "terminal" coroutine (it yields None).
682 """
683 lib = _reopen_lib(config.lib)
684 while True:
685 task = yield
686 if task.should_skip():
687 if config.resume is not False:
688 task.save_progress()
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
689 if config.incremental:
690 task.save_history()
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
691 continue
692
693 items = task.items if task.is_album else [task.item]
694
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
695 # Announce that we've added an album.
df6b1ab @sampsyo clean up vestiges of TRACKS choice for album tasks
sampsyo authored
696 if task.is_album:
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
697 album = lib.get_album(task.album_id)
698 plugins.send('album_imported', lib=lib, album=album)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
699 else:
df6b1ab @sampsyo clean up vestiges of TRACKS choice for album tasks
sampsyo authored
700 for item in items:
701 plugins.send('item_imported', lib=lib, item=item)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
702
703 # Finally, delete old files.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
704 if config.copy and config.delete:
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
705 new_paths = [os.path.realpath(item.path) for item in items]
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
706 for old_path in task.old_paths:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
707 # Only delete files that were actually moved.
708 if old_path not in new_paths:
e669868 @sampsyo move a bunch of functions to util
sampsyo authored
709 os.remove(syspath(old_path))
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
710
711 # Update progress.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
712 if config.resume is not False:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
713 task.save_progress()
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
714 if config.incremental:
715 task.save_history()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
716
717
45eef6e @sampsyo rename -i flag to -s ("singletons") and fix behavior
sampsyo authored
718 # Singleton pipeline stages.
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
719
720 def item_lookup(config):
721 """A coroutine used to perform the initial MusicBrainz lookup for
722 an item task.
723 """
724 task = None
725 while True:
726 task = yield task
a916551 @sampsyo progress/resume for item imports
sampsyo authored
727 if task.sentinel:
728 continue
729
704a76b @sampsyo rename interactive_autotag to timid
sampsyo authored
730 task.set_item_match(*autotag.tag_item(task.item, config.timid))
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
731
732 def item_query(config):
733 """A coroutine that queries the user for input on single-item
734 lookups.
735 """
7e89282 @sampsyo duplicate detection for items
sampsyo authored
736 lib = _reopen_lib(config.lib)
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
737 task = None
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
738 recent = set()
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
739 while True:
740 task = yield task
a916551 @sampsyo progress/resume for item imports
sampsyo authored
741 if task.sentinel:
742 continue
743
d1b7c8d @sampsyo display track changes; some scaffolding for user query
sampsyo authored
744 choice = config.choose_item_func(task, config)
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
745 task.set_choice(choice)
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
746 log_choice(config, task)
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
747
7e89282 @sampsyo duplicate detection for items
sampsyo authored
748 # Duplicate check.
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
749 if _item_duplicate_check(lib, task, recent):
750 tag_log(config.logfile, 'duplicate', task.item.path)
751 log.warn("This item is already in the library!")
752 task.set_choice(action.SKIP)
7e89282 @sampsyo duplicate detection for items
sampsyo authored
753
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
754 def item_progress(config):
755 """Skips the lookup and query stages in a non-autotagged singleton
756 import. Just shows progress.
757 """
758 task = None
759 log.info('Importing items:')
760 while True:
761 task = yield task
da6ee13 @sampsyo fix singleton quiet imports
sampsyo authored
762 if task.sentinel:
763 continue
764
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
765 log.info(task.item.path)
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
766 task.set_null_item_match()
767 task.set_choice(action.ASIS)
768
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
769
770 # Main driver.
771
772 def run_import(**kwargs):
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
773 """Run an import. The keyword arguments are the same as those to
774 ImportConfig.
775 """
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
776 config = ImportConfig(**kwargs)
777
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
778 # Set up the pipeline.
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
779 if config.query is None:
780 stages = [read_tasks(config)]
781 else:
782 stages = [query_tasks(config)]
45eef6e @sampsyo rename -i flag to -s ("singletons") and fix behavior
sampsyo authored
783 if config.singletons:
784 # Singleton importer.
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
785 if config.autot:
786 stages += [item_lookup(config), item_query(config)]
787 else:
788 stages += [item_progress(config)]
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
789 else:
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
790 # Whole-album importer.
791 if config.autot:
792 # Only look up and query the user when autotagging.
793 stages += [initial_lookup(config), user_query(config)]
794 else:
795 # When not autotagging, just display progress.
796 stages += [show_progress(config)]
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
797 stages += [apply_choices(config)]
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
798 if config.art:
799 stages += [fetch_art(config)]
800 stages += [finalize(config)]
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
801 pl = pipeline.Pipeline(stages)
802
803 # Run the pipeline.
804 try:
805 if config.threaded:
806 pl.run_parallel(QUEUE_SIZE)
807 else:
808 pl.run_sequential()
809 except ImportAbort:
810 # User aborted operation. Silently stop.
811 pass
Something went wrong with that request. Please try again.