Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 921 lines (799 sloc) 32.196 kB
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
1 # This file is part of beets.
2 # Copyright 2011, Adrian Sampson.
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining
5 # a copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish,
8 # distribute, sublicense, and/or sell copies of the Software, and to
9 # permit persons to whom the Software is furnished to do so, subject to
10 # the following conditions:
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
11 #
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
12 # The above copyright notice and this permission notice shall be
13 # included in all copies or substantial portions of the Software.
14
15 """Provides the basic, interface-agnostic workflow for importing and
16 autotagging music files.
17 """
429af42 @sampsyo use print_function __future__ import
sampsyo authored
18 from __future__ import print_function
19
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
20 import os
21 import logging
22 import pickle
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
23 from collections import defaultdict
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
24
25 from beets import autotag
26 from beets import library
27 from beets import plugins
153f52a @sampsyo import_delete prunes empty imported directories (#243)
sampsyo authored
28 from beets import util
2746f1f @sampsyo move pipeline into new beets.util package
sampsyo authored
29 from beets.util import pipeline
153f52a @sampsyo import_delete prunes empty imported directories (#243)
sampsyo authored
30 from beets.util import syspath, normpath, displayable_path
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
31 from beets.util.enumeration import enum
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
32
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
33 action = enum(
a074db7 @sampsyo manual specification of MBIDs
sampsyo authored
34 'SKIP', 'ASIS', 'TRACKS', 'MANUAL', 'APPLY', 'MANUAL_ID',
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
35 name='action'
36 )
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
37
38 QUEUE_SIZE = 128
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
39 STATE_FILE = os.path.expanduser('~/.beetsstate')
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
40 SINGLE_ARTIST_THRESH = 0.25
41 VARIOUS_ARTISTS = u'Various Artists'
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
42
43 # Global logger.
44 log = logging.getLogger('beets')
45
46 class ImportAbort(Exception):
47 """Raised when the user aborts the tagging operation.
48 """
49 pass
50
51
52 # Utilities.
53
54 def tag_log(logfile, status, path):
55 """Log a message about a given album to logfile. The status should
56 reflect the reason the album couldn't be tagged.
57 """
58 if logfile:
429af42 @sampsyo use print_function __future__ import
sampsyo authored
59 print('{0} {1}'.format(status, path), file=logfile)
531ebd1 @sampsyo import log: flush on write; close on crash (#337)
sampsyo authored
60 logfile.flush()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
61
34ec25e @sampsyo appropriate logging for duplicate resolution
sampsyo authored
62 def log_choice(config, task, duplicate=False):
63 """Logs the task's current choice if it should be logged. If
64 ``duplicate``, then this is a secondary choice after a duplicate was
65 detected and a decision was made.
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
66 """
67 path = task.path if task.is_album else task.item.path
34ec25e @sampsyo appropriate logging for duplicate resolution
sampsyo authored
68 if duplicate:
69 # Duplicate: log all three choices (skip, keep both, and trump).
70 if task.remove_duplicates:
71 tag_log(config.logfile, 'duplicate-replace', path)
72 elif task.choice_flag in (action.ASIS, action.APPLY):
73 tag_log(config.logfile, 'duplicate-keep', path)
74 elif task.choice_flag is (action.SKIP):
75 tag_log(config.logfile, 'duplicate-skip', path)
76 else:
77 # Non-duplicate: log "skip" and "asis" choices.
78 if task.choice_flag is action.ASIS:
79 tag_log(config.logfile, 'asis', path)
80 elif task.choice_flag is action.SKIP:
81 tag_log(config.logfile, 'skip', path)
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
82
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
83 def _duplicate_check(lib, task):
84 """Check whether an album already exists in the library. Returns a
85 list of Album objects (empty if no duplicates are found).
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
86 """
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
87 assert task.choice_flag in (action.ASIS, action.APPLY)
88 artist, album = task.chosen_ident()
58fb439 @sampsyo refactor duplicate tests to take the whole task as an argument
sampsyo authored
89
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
90 if artist is None:
91 # As-is import with no artist. Skip check.
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
92 return []
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
93
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
94 found_albums = []
9a7a551 @laarmen Enable import of incomplete albums
laarmen authored
95 cur_paths = set(i.path for i in task.items if i)
8341dee @sampsyo reorder items() and albums() parameters to reflect common use
sampsyo authored
96 for album_cand in lib.albums(artist=artist):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
97 if album_cand.album == album:
f313015 @sampsyo don't count existing items/albums as duplicates (allowing update)
sampsyo authored
98 # Check whether the album is identical in contents, in which
99 # case it is not a duplicate (will be replaced).
100 other_paths = set(i.path for i in album_cand.items())
101 if other_paths == cur_paths:
102 continue
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
103 found_albums.append(album_cand)
104 return found_albums
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
105
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
106 def _item_duplicate_check(lib, task):
107 """Check whether an item already exists in the library. Returns a
108 list of Item objects.
109 """
110 assert task.choice_flag in (action.ASIS, action.APPLY)
111 artist, title = task.chosen_ident()
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
112
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
113 found_items = []
d805401 @sampsyo simplifications afforded by eager result iterators (#261)
sampsyo authored
114 for other_item in lib.items(artist=artist, title=title):
115 # Existing items not considered duplicates.
116 if other_item.path == task.item.path:
117 continue
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
118 found_items.append(other_item)
119 return found_items
7e89282 @sampsyo duplicate detection for items
sampsyo authored
120
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
121 def _infer_album_fields(task):
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
122 """Given an album and an associated import task, massage the
123 album-level metadata. This ensures that the album artist is set
124 and that the "compilation" flag is set automatically.
125 """
126 assert task.is_album
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
127 assert task.items
128
129 changes = {}
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
130
131 if task.choice_flag == action.ASIS:
132 # Taking metadata "as-is". Guess whether this album is VA.
153f52a @sampsyo import_delete prunes empty imported directories (#243)
sampsyo authored
133 plur_artist, freq = util.plurality([i.artist for i in task.items])
951e4ee @sampsyo fix VA inference for small (1-track) albums
sampsyo authored
134 if freq == len(task.items) or (freq > 1 and
135 float(freq) / len(task.items) >= SINGLE_ARTIST_THRESH):
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
136 # Single-artist album.
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
137 changes['albumartist'] = plur_artist
138 changes['comp'] = False
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
139 else:
140 # VA.
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
141 changes['albumartist'] = VARIOUS_ARTISTS
142 changes['comp'] = True
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
143
144 elif task.choice_flag == action.APPLY:
145 # Applying autotagged metadata. Just get AA from the first
146 # item.
08b539a @sampsyo fix field inference w/ null first item (closes #14 on GitHub)
sampsyo authored
147 for item in task.items:
148 if item is not None:
149 first_item = item
150 break
151 else:
152 assert False, "all items are None"
153 if not first_item.albumartist:
154 changes['albumartist'] = first_item.artist
155 if not first_item.mb_albumartistid:
156 changes['mb_albumartistid'] = first_item.mb_artistid
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
157
158 else:
159 assert False
160
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
161 # Apply new metadata.
162 for item in task.items:
08b539a @sampsyo fix field inference w/ null first item (closes #14 on GitHub)
sampsyo authored
163 if item is not None:
164 for k, v in changes.iteritems():
165 setattr(item, k, v)
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
166
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
167 def _open_state():
168 """Reads the state file, returning a dictionary."""
169 try:
170 with open(STATE_FILE) as f:
171 return pickle.load(f)
5111537 @sampsyo handle EOFError when ~/.beetsstate is corrupted (#271)
sampsyo authored
172 except (IOError, EOFError):
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
173 return {}
174 def _save_state(state):
175 """Writes the state dictionary out to disk."""
5111537 @sampsyo handle EOFError when ~/.beetsstate is corrupted (#271)
sampsyo authored
176 try:
177 with open(STATE_FILE, 'w') as f:
178 pickle.dump(state, f)
760fff3 @sampsyo use new "except ... as ...:" syntax
sampsyo authored
179 except IOError as exc:
5111537 @sampsyo handle EOFError when ~/.beetsstate is corrupted (#271)
sampsyo authored
180 log.error(u'state file could not be written: %s' % unicode(exc))
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
181
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
182
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
183 # Utilities for reading and writing the beets progress file, which
184 # allows long tagging tasks to be resumed when they pause (or crash).
185 PROGRESS_KEY = 'tagprogress'
186 def progress_set(toppath, path):
187 """Record that tagging for the given `toppath` was successful up to
188 `path`. If path is None, then clear the progress value (indicating
189 that the tagging completed).
190 """
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
191 state = _open_state()
192 if PROGRESS_KEY not in state:
193 state[PROGRESS_KEY] = {}
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
194
195 if path is None:
196 # Remove progress from file.
197 if toppath in state[PROGRESS_KEY]:
198 del state[PROGRESS_KEY][toppath]
199 else:
200 state[PROGRESS_KEY][toppath] = path
201
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
202 _save_state(state)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
203 def progress_get(toppath):
204 """Get the last successfully tagged subpath of toppath. If toppath
205 has no progress information, returns None.
206 """
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
207 state = _open_state()
208 if PROGRESS_KEY not in state:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
209 return None
210 return state[PROGRESS_KEY].get(toppath)
211
212
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
213 # Similarly, utilities for manipulating the "incremental" import log.
214 # This keeps track of all directories that were ever imported, which
215 # allows the importer to only import new stuff.
216 HISTORY_KEY = 'taghistory'
217 def history_add(path):
218 """Indicate that the import of `path` is completed and should not
219 be repeated in incremental imports.
220 """
221 state = _open_state()
222 if HISTORY_KEY not in state:
223 state[HISTORY_KEY] = set()
224
225 state[HISTORY_KEY].add(path)
226
227 _save_state(state)
228 def history_get():
229 """Get the set of completed paths in incremental imports.
230 """
231 state = _open_state()
232 if HISTORY_KEY not in state:
233 return set()
234 return state[HISTORY_KEY]
235
236
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
237 # The configuration structure.
238
239 class ImportConfig(object):
240 """Contains all the settings used during an import session. Should
241 be used in a "write-once" way -- everything is set up initially and
242 then never touched again.
243 """
6884fc2 @sampsyo don't use __slots__
sampsyo authored
244 _fields = ['lib', 'paths', 'resume', 'logfile', 'color', 'quiet',
11d4fb1 @sampsyo move album art fetching to a plugin (fetchart)
sampsyo authored
245 'quiet_fallback', 'copy', 'move', 'write', 'delete',
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
246 'choose_match_func', 'should_resume_func', 'threaded',
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
247 'autot', 'singletons', 'timid', 'choose_item_func',
19b08f8 @sampsyo duplicate resolution callback function (#164)
sampsyo authored
248 'query', 'incremental', 'ignore',
2b000c4 @sampsyo per_disc_numbering config option (GC-335)
sampsyo authored
249 'resolve_duplicate_func', 'per_disc_numbering']
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
250 def __init__(self, **kwargs):
6884fc2 @sampsyo don't use __slots__
sampsyo authored
251 for slot in self._fields:
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
252 setattr(self, slot, kwargs[slot])
253
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
254 # Normalize the paths.
255 if self.paths:
256 self.paths = map(normpath, self.paths)
257
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
258 # Incremental and progress are mutually exclusive.
259 if self.incremental:
260 self.resume = False
261
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
262 # When based on a query instead of directories, never
263 # save progress or try to resume.
264 if self.query is not None:
265 self.paths = None
266 self.resume = False
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
267 self.incremental = False
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
268
6b696c8 @sampsyo cleanup and docs for import_move (GH-26, GC-266)
sampsyo authored
269 # Copy and move are mutually exclusive.
270 if self.move:
271 self.copy = False
272
273 # Only delete when copying.
274 if not self.copy:
275 self.delete = False
276
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
277
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
278 # The importer task class.
279
280 class ImportTask(object):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
281 """Represents a single set of items to be imported along with its
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
282 intermediate state. May represent an album or a single item.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
283 """
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
284 def __init__(self, toppath=None, path=None, items=None):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
285 self.toppath = toppath
286 self.path = path
287 self.items = items
288 self.sentinel = False
ced4e1a @sampsyo duplicate resolution prompt (#164)
sampsyo authored
289 self.remove_duplicates = False
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
290 self.is_album = True
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
291
292 @classmethod
293 def done_sentinel(cls, toppath):
294 """Create an ImportTask that indicates the end of a top-level
295 directory import.
296 """
297 obj = cls(toppath)
298 obj.sentinel = True
299 return obj
300
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
301 @classmethod
a916551 @sampsyo progress/resume for item imports
sampsyo authored
302 def progress_sentinel(cls, toppath, path):
303 """Create a task indicating that a single directory in a larger
304 import has finished. This is only required for singleton
305 imports; progress is implied for album imports.
306 """
307 obj = cls(toppath, path)
308 obj.sentinel = True
309 return obj
310
311 @classmethod
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
312 def item_task(cls, item):
313 """Creates an ImportTask for a single item."""
314 obj = cls()
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
315 obj.item = item
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
316 obj.is_album = False
317 return obj
318
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
319 def set_candidates(self, cur_artist, cur_album, candidates, rec):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
320 """Sets the candidates for this album matched by the
321 `autotag.tag_album` method.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
322 """
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
323 assert self.is_album
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
324 assert not self.sentinel
325 self.cur_artist = cur_artist
326 self.cur_album = cur_album
327 self.candidates = candidates
328 self.rec = rec
329
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
330 def set_null_candidates(self):
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
331 """Set the candidates to indicate no album match was found.
332 """
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
333 self.cur_artist = None
334 self.cur_album = None
335 self.candidates = None
336 self.rec = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
337
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
338 def set_item_candidates(self, candidates, rec):
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
339 """Set the match for a single-item task."""
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
340 assert not self.is_album
341 assert self.item is not None
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
342 self.candidates = candidates
343 self.rec = rec
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
344
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
345 def set_choice(self, choice):
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
346 """Given an AlbumMatch or TrackMatch object or an action constant,
347 indicates that an action has been selected for this task.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
348 """
349 assert not self.sentinel
a074db7 @sampsyo manual specification of MBIDs
sampsyo authored
350 # Not part of the task structure:
351 assert choice not in (action.MANUAL, action.MANUAL_ID)
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
352 assert choice != action.APPLY # Only used internally.
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
353 if choice in (action.SKIP, action.ASIS, action.TRACKS):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
354 self.choice_flag = choice
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
355 self.match = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
356 else:
86b1f92 @sampsyo fix some (suprisingly few) airplane-coding bugs
sampsyo authored
357 if self.is_album:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
358 assert isinstance(choice, autotag.AlbumMatch)
86b1f92 @sampsyo fix some (suprisingly few) airplane-coding bugs
sampsyo authored
359 else:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
360 assert isinstance(choice, autotag.TrackMatch)
361 self.choice_flag = action.APPLY # Implicit choice.
362 self.match = choice
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
363
364 def save_progress(self):
365 """Updates the progress state to indicate that this album has
366 finished.
367 """
a916551 @sampsyo progress/resume for item imports
sampsyo authored
368 if self.sentinel and self.path is None:
369 # "Done" sentinel.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
370 progress_set(self.toppath, None)
a916551 @sampsyo progress/resume for item imports
sampsyo authored
371 elif self.sentinel or self.is_album:
372 # "Directory progress" sentinel for singletons or a real
373 # album task, which implies the same.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
374 progress_set(self.toppath, self.path)
375
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
376 def save_history(self):
377 """Save the directory in the history for incremental imports.
378 """
379 if self.sentinel or self.is_album:
380 history_add(self.path)
381
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
382
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
383 # Logical decisions.
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
384
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
385 def should_write_tags(self):
386 """Should new info be written to the files' metadata?"""
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
387 if self.choice_flag == action.APPLY:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
388 return True
a675988 @sampsyo add and use fancy enumeration module
sampsyo authored
389 elif self.choice_flag in (action.ASIS, action.TRACKS, action.SKIP):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
390 return False
391 else:
392 assert False
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
393
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
394 def should_skip(self):
395 """After a choice has been made, returns True if this is a
396 sentinel or it has been marked for skipping.
397 """
398 return self.sentinel or self.choice_flag == action.SKIP
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
399
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
400
401 # Convenient data.
402
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
403 def chosen_ident(self):
404 """Returns identifying metadata about the current choice. For
405 albums, this is an (artist, album) pair. For items, this is
406 (artist, title). May only be called when the choice flag is ASIS
407 (in which case the data comes from the files' current metadata)
408 or APPLY (data comes from the choice).
409 """
410 assert self.choice_flag in (action.ASIS, action.APPLY)
411 if self.is_album:
412 if self.choice_flag is action.ASIS:
413 return (self.cur_artist, self.cur_album)
414 elif self.choice_flag is action.APPLY:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
415 return (self.match.info.artist, self.match.info.album)
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
416 else:
417 if self.choice_flag is action.ASIS:
418 return (self.item.artist, self.item.title)
419 elif self.choice_flag is action.APPLY:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
420 return (self.match.info.artist, self.match.info.title)
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
421
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
422 def imported_items(self):
423 """Return a list of Items that should be added to the library.
424 If this is an album task, return the list of items in the
425 selected match or everything if the choice is ASIS. If this is a
426 singleton task, return a list containing the item.
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
427 """
428 if self.is_album:
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
429 if self.choice_flag == action.ASIS:
430 return list(self.items)
431 elif self.choice_flag == action.APPLY:
432 return self.match.mapping.keys()
433 else:
434 assert False
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
435 else:
436 return [self.item]
437
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
438
a2f4940 @sampsyo factor out ImportTask.prune() utility function
sampsyo authored
439 # Utilities.
440
441 def prune(self, filename):
e00f151 @sampsyo prune is a no-op when file exists
sampsyo authored
442 """Prune any empty directories above the given file. If this
443 task has no `toppath` or the file path provided is not within
444 the `toppath`, then this function has no effect. Similarly, if
445 the file still exists, no pruning is performed, so it's safe to
446 call when the file in question may not have been removed.
a2f4940 @sampsyo factor out ImportTask.prune() utility function
sampsyo authored
447 """
e00f151 @sampsyo prune is a no-op when file exists
sampsyo authored
448 if self.toppath and not os.path.exists(filename):
a2f4940 @sampsyo factor out ImportTask.prune() utility function
sampsyo authored
449 util.prune_dirs(os.path.dirname(filename), self.toppath)
450
451
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
452 # Full-album pipeline stages.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
453
a916551 @sampsyo progress/resume for item imports
sampsyo authored
454 def read_tasks(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
455 """A generator yielding all the albums (as ImportTask objects) found
3b23198 @sampsyo singleton imports can take single-file arguments (#184)
sampsyo authored
456 in the user-specified list of paths. In the case of a singleton
457 import, yields single-item tasks instead.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
458 """
459 # Look for saved progress.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
460 progress = config.resume is not False
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
461 if progress:
462 resume_dirs = {}
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
463 for path in config.paths:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
464 resume_dir = progress_get(path)
465 if resume_dir:
466
467 # Either accept immediately or prompt for input to decide.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
468 if config.resume:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
469 do_resume = True
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
470 log.warn('Resuming interrupted import of %s' % path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
471 else:
dffdbce @sampsyo sanitize CLI-specific code from importer module
sampsyo authored
472 do_resume = config.should_resume_func(config, path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
473
474 if do_resume:
475 resume_dirs[path] = resume_dir
476 else:
477 # Clear progress; we're starting from the top.
478 progress_set(path, None)
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
479
480 # Look for saved incremental directories.
481 if config.incremental:
6fff1b9 @sampsyo message when skipping directories in incremental mode (#273)
sampsyo authored
482 incremental_skipped = 0
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
483 history_dirs = history_get()
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
484
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
485 for toppath in config.paths:
3b23198 @sampsyo singleton imports can take single-file arguments (#184)
sampsyo authored
486 # Check whether the path is to a file.
487 if config.singletons and not os.path.isdir(syspath(toppath)):
488 item = library.Item.from_path(toppath)
489 yield ImportTask.item_task(item)
490 continue
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
491
3b23198 @sampsyo singleton imports can take single-file arguments (#184)
sampsyo authored
492 # Produce paths under this directory.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
493 if progress:
494 resume_dir = resume_dirs.get(toppath)
5965b37 @sampsyo skip (configurable) clutter filenames when importing
sampsyo authored
495 for path, items in autotag.albums_in_dir(toppath, config.ignore):
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
496 # Skip according to progress.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
497 if progress and resume_dir:
498 # We're fast-forwarding to resume a previous tagging.
499 if path == resume_dir:
500 # We've hit the last good path! Turn off the
501 # fast-forwarding.
502 resume_dir = None
503 continue
504
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
505 # When incremental, skip paths in the history.
506 if config.incremental and path in history_dirs:
6fff1b9 @sampsyo message when skipping directories in incremental mode (#273)
sampsyo authored
507 log.debug(u'Skipping previously-imported path: %s' %
508 displayable_path(path))
509 incremental_skipped += 1
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
510 continue
511
a916551 @sampsyo progress/resume for item imports
sampsyo authored
512 # Yield all the necessary tasks.
513 if config.singletons:
514 for item in items:
515 yield ImportTask.item_task(item)
516 yield ImportTask.progress_sentinel(toppath, path)
517 else:
518 yield ImportTask(toppath, path, items)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
519
520 # Indicate the directory is finished.
521 yield ImportTask.done_sentinel(toppath)
522
6fff1b9 @sampsyo message when skipping directories in incremental mode (#273)
sampsyo authored
523 # Show skipped directories.
524 if config.incremental and incremental_skipped:
525 log.info(u'Incremental import: skipped %i directories.' %
526 incremental_skipped)
527
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
528 def query_tasks(config):
529 """A generator that works as a drop-in-replacement for read_tasks.
530 Instead of finding files from the filesystem, a query is used to
531 match items from the library.
532 """
533 if config.singletons:
534 # Search for items.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
535 for item in config.lib.items(config.query):
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
536 yield ImportTask.item_task(item)
537
538 else:
539 # Search for albums.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
540 for album in config.lib.albums(config.query):
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
541 log.debug('yielding album %i: %s - %s' %
542 (album.id, album.albumartist, album.album))
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
543 items = list(album.items())
544 yield ImportTask(None, album.item_dir(), items)
545
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
546 def initial_lookup(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
547 """A coroutine for performing the initial MusicBrainz lookup for an
548 album. It accepts lists of Items and yields
549 (items, cur_artist, cur_album, candidates, rec) tuples. If no match
550 is found, all of the yielded parameters (except items) are None.
551 """
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
552 task = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
553 while True:
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
554 task = yield task
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
555 if task.sentinel:
556 continue
557
01dce53 @sampsyo store Acoustid data in DB & file
sampsyo authored
558 plugins.send('import_task_start', task=task, config=config)
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
559
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
560 log.debug('Looking up: %s' % task.path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
561 try:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
562 task.set_candidates(*autotag.tag_album(task.items, config.timid))
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
563 except autotag.AutotagError:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
564 task.set_null_candidates()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
565
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
566 def user_query(config):
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
567 """A coroutine for interfacing with the user about the tagging
568 process. lib is the Library to import into and logfile may be
569 a file-like object for logging the import process. The coroutine
570 accepts and yields ImportTask objects.
571 """
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
572 recent = set()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
573 task = None
574 while True:
575 task = yield task
576 if task.sentinel:
577 continue
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
578
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
579 # Ask the user for a choice.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
580 choice = config.choose_match_func(task, config)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
581 task.set_choice(choice)
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
582 log_choice(config, task)
c9da7bf @sampsyo new plugin event: import_task_choice
sampsyo authored
583 plugins.send('import_task_choice', task=task, config=config)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
584
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
585 # As-tracks: transition to singleton workflow.
586 if choice is action.TRACKS:
587 # Set up a little pipeline for dealing with the singletons.
588 item_tasks = []
1b3e7d9 @sampsyo fix singleton invocation of pipeline
sampsyo authored
589 def emitter():
590 for item in task.items:
591 yield ImportTask.item_task(item)
a916551 @sampsyo progress/resume for item imports
sampsyo authored
592 yield ImportTask.progress_sentinel(task.toppath, task.path)
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
593 def collector():
594 while True:
595 item_task = yield
596 item_tasks.append(item_task)
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
597 ipl = pipeline.Pipeline((emitter(), item_lookup(config),
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
598 item_query(config), collector()))
599 ipl.run_sequential()
600 task = pipeline.multiple(item_tasks)
c891dac @sampsyo fix crash with "as Tracks" import option (#244)
sampsyo authored
601 continue
3a5f9ec @sampsyo TRACKS selection now runs singleton pipeline and forwards
sampsyo authored
602
603 # Check for duplicates if we have a match (or ASIS).
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
604 if task.choice_flag in (action.ASIS, action.APPLY):
605 ident = task.chosen_ident()
606 # The "recent" set keeps track of identifiers for recently
607 # imported albums -- those that haven't reached the database
608 # yet.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
609 if ident in recent or _duplicate_check(config.lib, task):
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
610 config.resolve_duplicate_func(task, config)
611 log_choice(config, task, True)
612 recent.add(ident)
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
613
614 def show_progress(config):
615 """This stage replaces the initial_lookup and user_query stages
616 when the importer is run without autotagging. It displays the album
617 name and artist as the files are added.
618 """
619 task = None
620 while True:
621 task = yield task
622 if task.sentinel:
623 continue
624
625 log.info(task.path)
626
627 # Behave as if ASIS were selected.
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
628 task.set_null_candidates()
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
629 task.set_choice(action.ASIS)
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
630
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
631 def apply_choices(config):
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
632 """A coroutine for applying changes to albums and singletons during
633 the autotag process.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
634 """
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
635 task = None
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
636 while True:
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
637 task = yield task
638 if task.should_skip():
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
639 continue
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
640
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
641 items = task.imported_items()
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
642 # Clear IDs in case the items are being re-tagged.
643 for item in items:
644 item.id = None
2e066b8 @sampsyo moving when re-importing reassociates albums before renaming
sampsyo authored
645 item.album_id = None
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
646
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
647 # Change metadata.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
648 if task.should_write_tags():
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
649 if task.is_album:
2b000c4 @sampsyo per_disc_numbering config option (GC-335)
sampsyo authored
650 autotag.apply_metadata(
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
651 task.match.info, task.match.mapping,
2b000c4 @sampsyo per_disc_numbering config option (GC-335)
sampsyo authored
652 per_disc_numbering=config.per_disc_numbering
653 )
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
654 else:
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
655 autotag.apply_item_metadata(task.item, task.match.info)
01dce53 @sampsyo store Acoustid data in DB & file
sampsyo authored
656 plugins.send('import_task_apply', config=config, task=task)
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
657
45383ec @sampsyo fix VA inference (needs to run before move step in "apply")
sampsyo authored
658 # Infer album-level fields.
659 if task.is_album:
660 _infer_album_fields(task)
661
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
662 # Find existing item entries that these are replacing (for
663 # re-imports). Old album structures are automatically cleaned up
664 # when the last item is removed.
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
665 task.replaced_items = defaultdict(list)
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
666 for item in items:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
667 dup_items = config.lib.items(library.MatchQuery('path', item.path))
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
668 for dup_item in dup_items:
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
669 task.replaced_items[item].append(dup_item)
948f7ef @sampsyo fix logging of unicode pathnames
sampsyo authored
670 log.debug('replacing item %i: %s' %
671 (dup_item.id, displayable_path(item.path)))
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
672 log.debug('%i of %i items replaced' % (len(task.replaced_items),
2c56fd2 @sampsyo fix replacement of in-library items
sampsyo authored
673 len(items)))
248bccf @sampsyo move, rather than copying, when re-importing
sampsyo authored
674
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
675 # Find old items that should be replaced as part of a duplicate
676 # resolution.
677 duplicate_items = []
678 if task.remove_duplicates:
679 if task.is_album:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
680 for album in _duplicate_check(config.lib, task):
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
681 duplicate_items += album.items()
682 else:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
683 duplicate_items = _item_duplicate_check(config.lib, task)
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
684 log.debug('removing %i old duplicated items' %
685 len(duplicate_items))
686
687 # Delete duplicate files that are located inside the library
688 # directory.
689 for duplicate_path in [i.path for i in duplicate_items]:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
690 if config.lib.directory in util.ancestry(duplicate_path):
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
691 log.debug(u'deleting replaced duplicate %s' %
692 util.displayable_path(duplicate_path))
1399520 @sampsyo human-readable filesystem errors (#387)
sampsyo authored
693 util.remove(duplicate_path)
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
694 util.prune_dirs(os.path.dirname(duplicate_path),
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
695 config.lib.directory)
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
696
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
697 # Add items -- before path changes -- to the library. We add the
698 # items now (rather than at the end) so that album structures
699 # are in place before calls to destination().
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
700 with config.lib.transaction():
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
701 # Remove old items.
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
702 for replaced in task.replaced_items.itervalues():
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
703 for item in replaced:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
704 config.lib.remove(item)
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
705 for item in duplicate_items:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
706 config.lib.remove(item)
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
707
708 # Add new ones.
709 if task.is_album:
710 # Add an album.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
711 album = config.lib.add_album(items)
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
712 task.album_id = album.id
713 else:
714 # Add tracks.
715 for item in items:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
716 config.lib.add(item)
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
717
48ffa08 @sampsyo plugin import stages
sampsyo authored
718 def plugin_stage(config, func):
719 """A coroutine (pipeline stage) that calls the given function with
720 each non-skipped import task. These stages occur between applying
721 metadata changes and moving/copying/writing files.
722 """
723 task = None
724 while True:
725 task = yield task
726 if task.should_skip():
727 continue
728 func(config, task)
729
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
730 def manipulate_files(config):
731 """A coroutine (pipeline stage) that performs necessary file
732 manipulations *after* items have been added to the library.
733 """
734 task = None
735 while True:
736 task = yield task
737 if task.should_skip():
738 continue
739
a448879 @sampsyo infer album artist or VA for as-is imports (#161)
sampsyo authored
740 # Move/copy files.
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
741 items = task.imported_items()
6b696c8 @sampsyo cleanup and docs for import_move (GH-26, GC-266)
sampsyo authored
742 task.old_paths = [item.path for item in items] # For deletion.
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
743 for item in items:
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
744 if config.move:
745 # Just move the file.
746 old_path = item.path
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
747 config.lib.move(item, False)
e00f151 @sampsyo prune is a no-op when file exists
sampsyo authored
748 task.prune(old_path)
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
749 elif config.copy:
750 # If it's a reimport, move in-library files and copy
751 # out-of-library files. Otherwise, copy and keep track
752 # of the old path.
753 old_path = item.path
86f513d @sampsyo split apply_choice coroutine
sampsyo authored
754 if task.replaced_items[item]:
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
755 # This is a reimport. Move in-library files and copy
756 # out-of-library files.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
757 if config.lib.directory in util.ancestry(old_path):
758 config.lib.move(item, False)
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
759 # We moved the item, so remove the
760 # now-nonexistent file from old_paths.
6b696c8 @sampsyo cleanup and docs for import_move (GH-26, GC-266)
sampsyo authored
761 task.old_paths.remove(old_path)
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
762 else:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
763 config.lib.move(item, True)
a6c1ad2 @sampsyo reimporting with copying: copy external files
sampsyo authored
764 else:
765 # A normal import. Just copy files and keep track of
766 # old paths.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
767 config.lib.move(item, True)
1af4f86 @domenkozar support move action when importing
domenkozar authored
768
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
769 if config.write and task.should_write_tags():
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
770 item.write()
771
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
772 # Save new paths.
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
773 with config.lib.transaction():
2087ff6 @sampsyo add items to DB before moving/copying (#190)
sampsyo authored
774 for item in items:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
775 config.lib.store(item)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
776
ad4b7f8 @sampsyo new plugin event: import_task_files
sampsyo authored
777 # Plugin event.
778 plugins.send('import_task_files', config=config, task=task)
779
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
780 def finalize(config):
781 """A coroutine that finishes up importer tasks. In particular, the
782 coroutine sends plugin events, deletes old files, and saves
783 progress. This is a "terminal" coroutine (it yields None).
784 """
785 while True:
786 task = yield
787 if task.should_skip():
788 if config.resume is not False:
789 task.save_progress()
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
790 if config.incremental:
791 task.save_history()
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
792 continue
793
c5f28ec @sampsyo don't import unmatched tracks
sampsyo authored
794 items = task.imported_items()
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
795
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
796 # Announce that we've added an album.
df6b1ab @sampsyo clean up vestiges of TRACKS choice for album tasks
sampsyo authored
797 if task.is_album:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
798 album = config.lib.get_album(task.album_id)
799 plugins.send('album_imported',
800 lib=config.lib, album=album, config=config)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
801 else:
df6b1ab @sampsyo clean up vestiges of TRACKS choice for album tasks
sampsyo authored
802 for item in items:
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
803 plugins.send('item_imported',
804 lib=config.lib, item=item, config=config)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
805
806 # Finally, delete old files.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
807 if config.copy and config.delete:
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
808 new_paths = [os.path.realpath(item.path) for item in items]
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
809 for old_path in task.old_paths:
c400818 @sampsyo fix double-removal when re-importing with deletion
sampsyo authored
810 # Only delete files that were actually copied.
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
811 if old_path not in new_paths:
1399520 @sampsyo human-readable filesystem errors (#387)
sampsyo authored
812 util.remove(syspath(old_path), False)
e00f151 @sampsyo prune is a no-op when file exists
sampsyo authored
813 task.prune(old_path)
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
814
815 # Update progress.
c0467c3 @sampsyo encapsulate importer configuration in an object
sampsyo authored
816 if config.resume is not False:
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
817 task.save_progress()
607757e @sampsyo -i/import_incremental to only import new directories (#99)
sampsyo authored
818 if config.incremental:
819 task.save_history()
2339252 @sampsyo The Great Importer Refactoring
sampsyo authored
820
821
45eef6e @sampsyo rename -i flag to -s ("singletons") and fix behavior
sampsyo authored
822 # Singleton pipeline stages.
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
823
824 def item_lookup(config):
825 """A coroutine used to perform the initial MusicBrainz lookup for
826 an item task.
827 """
828 task = None
829 while True:
830 task = yield task
a916551 @sampsyo progress/resume for item imports
sampsyo authored
831 if task.sentinel:
832 continue
833
01dce53 @sampsyo store Acoustid data in DB & file
sampsyo authored
834 plugins.send('import_task_start', task=task, config=config)
82a4baf @sampsyo chroma: fingerprint when task begins
sampsyo authored
835
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
836 task.set_item_candidates(*autotag.tag_item(task.item, config.timid))
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
837
838 def item_query(config):
839 """A coroutine that queries the user for input on single-item
840 lookups.
841 """
842 task = None
a0ef39a @sampsyo duplicate detection on adjacent albums/items now works (#156)
sampsyo authored
843 recent = set()
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
844 while True:
845 task = yield task
a916551 @sampsyo progress/resume for item imports
sampsyo authored
846 if task.sentinel:
847 continue
848
d1b7c8d @sampsyo display track changes; some scaffolding for user query
sampsyo authored
849 choice = config.choose_item_func(task, config)
bf5c569 @sampsyo simplify ImportTask for singletons: use .item instead of items[0]
sampsyo authored
850 task.set_choice(choice)
b80b0b4 @sampsyo logging for singleton imports
sampsyo authored
851 log_choice(config, task)
096cf8b @sampsyo import_task_choice plugin event for singletons
sampsyo authored
852 plugins.send('import_task_choice', task=task, config=config)
a39a5b5 @sampsyo extremely preliminary item importer skeleton
sampsyo authored
853
7e89282 @sampsyo duplicate detection for items
sampsyo authored
854 # Duplicate check.
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
855 if task.choice_flag in (action.ASIS, action.APPLY):
856 ident = task.chosen_ident()
00c47b6 @sampsyo library has per-thread transaction stacks (GC-399)
sampsyo authored
857 if ident in recent or _item_duplicate_check(config.lib, task):
24cdf2a @sampsyo duplicate trumping: remove items & delete files
sampsyo authored
858 config.resolve_duplicate_func(task, config)
859 log_choice(config, task, True)
860 recent.add(ident)
7e89282 @sampsyo duplicate detection for items
sampsyo authored
861
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
862 def item_progress(config):
863 """Skips the lookup and query stages in a non-autotagged singleton
864 import. Just shows progress.
865 """
866 task = None
867 log.info('Importing items:')
868 while True:
869 task = yield task
da6ee13 @sampsyo fix singleton quiet imports
sampsyo authored
870 if task.sentinel:
871 continue
872
948f7ef @sampsyo fix logging of unicode pathnames
sampsyo authored
873 log.info(displayable_path(task.item.path))
ce16600 @sampsyo use AlbumMatch/TrackMatch objects everywhere
sampsyo authored
874 task.set_null_candidates()
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
875 task.set_choice(action.ASIS)
876
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
877
878 # Main driver.
879
880 def run_import(**kwargs):
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
881 """Run an import. The keyword arguments are the same as those to
882 ImportConfig.
883 """
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
884 config = ImportConfig(**kwargs)
b68e87b @sampsyo The Great Trailing Whitespace Purge of 2012
sampsyo authored
885
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
886 # Set up the pipeline.
3efeb9a @sampsyo -L flag to import lets you re-import items matching query (#69)
sampsyo authored
887 if config.query is None:
888 stages = [read_tasks(config)]
889 else:
890 stages = [query_tasks(config)]
45eef6e @sampsyo rename -i flag to -s ("singletons") and fix behavior
sampsyo authored
891 if config.singletons:
892 # Singleton importer.
8df852d @sampsyo non-autotagged singleton import
sampsyo authored
893 if config.autot:
894 stages += [item_lookup(config), item_query(config)]
895 else:
896 stages += [item_progress(config)]
3222cc6 @sampsyo move main importer driver logic to importer module
sampsyo authored
897 else:
12854ad @sampsyo very first stab at a working individual-item importer flow
sampsyo authored
898 # Whole-album importer.
899 if config.autot:
900 # Only look up and query the user when autotagging.
901 stages += [initial_lookup(config), user_query(config)]
902 else:
903 # When not autotagging, just display progress.
904 stages += [show_progress(config)]
48ffa08 @sampsyo plugin import stages
sampsyo authored
905 stages += [apply_choices(config)]
906 for stage_func in plugins.import_stages():
907 stages.append(plugin_stage(config, stage_func))
908 stages += [manipulate_files(config)]
65dac30 @sampsyo break album art and finalization into new stages (#168)
sampsyo authored
909 stages += [finalize(config)]
fcee8b2 @sampsyo remove special-cased non-autotagged import function (simple_import)
sampsyo authored
910 pl = pipeline.Pipeline(stages)
911
912 # Run the pipeline.
913 try:
914 if config.threaded:
915 pl.run_parallel(QUEUE_SIZE)
916 else:
917 pl.run_sequential()
918 except ImportAbort:
919 # User aborted operation. Silently stop.
920 pass
Something went wrong with that request. Please try again.