plugin cd: Replace ccdb with libdiscid+musicbrainz (#824)

* Add musicbrainzngs
* Put all the linux-specific code into a new module
  * This module directly accesses the kernel using ioctl
  * Completely rewrite it for readability
* Main module:
  * Add platform detection
  * Add metadata provider hierarchy
* Add a new module for parsing discid data
* Add a new module for parsing musicbrainzngs data based on discid
* Main module [WIP]:
  * Disable threading support for now
  * incomplete discid+musicbrainzngs parsing missing special cases
* linux_cd_parser:
  * Add code for reading MCN
  * Fix track number format
  * cleanup
* discid_parser:
  * Read extra features like mcn and isrc if possible
  * Fix track number format
  * Save more data to track metadata
* musicbrainzngs_parser.py:
  * implement cdstub parsing
  * improve disc parsing
    * calculate priorities from how good the data matches the disc
    * implement hierarchical priority and filter
    * filter available data based on priorities
* discid_parser.py:
  * Split read function out to make it accessible from __init__.py
* __init__.py:
  * Add more ideas
  * Improve data provider logic
* make musicbrainzngs parser rely on discid
* musicbrainzngs_parser.py:
  * Pre populate tracks from discid_parser
    * This includes low-level fields such as MCN, ISRC, FreeDB ID, Musicbrainz ID, tracknumber, length
  * Add links to documentation
  * Simplify user agent initialization
  * Fetch more fields from musicbrainz:
    * isrcs -> improve detection of right metadata set
    * artist-credits -> include track specific artists or multiple artists
  * improve parser syntax for readability
  * change log level, remove some debugging code and comments
  * fetch cover image
* Move setting of musicbrainz_albumid to discid_parser
* Introduce and fix multi-threading
  * Both disc I/O and fetching metadata from the internet happens async now.
* Add option to disable metadata fetch from internet
* Improve callbacks, always invoke them
  * Rationale: Needed to be able to abort progress indication
* Save isrc tag in a way compatible to Picard
* Set musicbrainz useragent each time before making an api call since
  we're using it for multiple plugins.

Co-authored-by: Christian Stadelmann <dev@genodeftest.de>

DEPS

@@ -54,7 +54,8 @@ Device detection:
 
 CD info: (TODO: This is currently broken on python3, see #608 and #652)
 
-* cddb (python2), from http://cddb-py.sourceforge.net/
+* python-libdiscid or python-discid (optional on linux, required to use musicbrainz)
+* python-musicbrainzngs (optional)
 
 DAAP plugins (daapserver and daapclient):
 

plugins/cd/PLUGININFO

@@ -1,7 +1,7 @@
 Version='4.1.2'
 Authors=['Aren Olson <reacocard@gmail.com>']
 Name=_('CD Playback')
-Description=_('Adds support for playing audio CDs.\n\nRequires UDisks2 to autodetect CDs\nRequires cddb-py (%s) to look up tags.') % 'http://cddb-py.sourceforge.net/'
+Description=_('Adds support for playing audio CDs.\n\nRequires UDisks2 to autodetect CDs\nRequires python-libdiscid (%s) and python-musicbrainzngs (%s) to look up tags.') % ('https://pypi.org/project/python-libdiscid/', 'https://pypi.org/project/musicbrainzngs/')
 Category=_('Devices')
 Platforms=['linux']
 RequiredModules=['dbus']

plugins/cd/__init__.py

@@ -25,44 +25,53 @@
 # from your version.
 
 import dbus
-from fcntl import ioctl
+import importlib
 import logging
-import os
-import struct
+import os.path
+import sys
+
+from gi.repository import GLib
 
 from xl.nls import gettext as _
-from xl import providers, event
+from xl import providers, event, main
 from xl.hal import Handler, UDisksProvider
 from xl.devices import Device, KeyedDevice
+from xl import playlist, trax, common, settings
+from xl.trax import Track
 
-from xl import playlist, trax, common
-import os.path
-
-from . import cdprefs
-
-try:
-    import DiscID
-    import CDDB
+from . import cdprefs, _cdguipanel
+from sys import exc_info
 
-    CDDB_AVAIL = True
-except Exception:
-    CDDB_AVAIL = False
 
 logger = logging.getLogger(__name__)
 
 
-TOC_HEADER_FMT = 'BB'
-TOC_ENTRY_FMT = 'BBBix'
-ADDR_FMT = 'BBB' + 'x' * (struct.calcsize('i') - 3)
-CDROMREADTOCHDR = 0x5305
-CDROMREADTOCENTRY = 0x5306
-CDROM_LEADOUT = 0xAA
-CDROM_MSF = 0x02
-CDROM_DATA_TRACK = 0x04
+if sys.platform.startswith('linux'):
+    from . import linux_cd_parser
 
 
 class CdPlugin:
+    discid_parser = None
+    musicbrainzngs_parser = None
+
+    def __import_dependency(self, module_name):
+        try:
+            full_name = 'plugins.cd.' + module_name + '_parser'
+            return importlib.import_module(full_name)
+        except ImportError:
+            logger.warn(
+                'Cannot import optional dependency "%s" for plugin cd.', module_name
+            )
+            logger.debug(
+                'Traceback for importing dependency "%s" for plugin cd.',
+                module_name,
+                exc_info=True,
+            )
+            return None
+
     def enable(self, exaile):
+        CdPlugin.discid_parser = self.__import_dependency('discid')
+        CdPlugin.musicbrainzngs_parser = self.__import_dependency('musicbrainzngs')
         self.__exaile = exaile
         self.__udisks2 = None
 
@@ -156,59 +165,92 @@ def __init__(self, name=_("Audio Disc"), device=None):
             self.__device = "/dev/cdrom"
         else:
             self.__device = device
-
-        self.open_disc()
-
-    def open_disc(self):
-
-        toc = CDTocParser(self.__device)
-        lengths = toc._get_track_lengths()
-
-        songs = []
-
-        for count, length in enumerate(lengths):
-            count += 1
-            song = trax.Track("cdda://%d/#%s" % (count, self.__device))
-            song.set_tags(
-                title="Track %d" % count, tracknumber=str(count), __length=length
-            )
-            songs.append(song)
-
-        self.extend(songs)
-
-        if CDDB_AVAIL:
-            self.get_cddb_info()
+        self.__read_disc_index_async(device)
 
     @common.threaded
-    def get_cddb_info(self):
-        try:
-            disc = DiscID.open(self.__device)
-            self.__device = DiscID.disc_id(disc)
-            status, info = CDDB.query(self.__device)
-        except IOError:
-            return
-
-        if status in (210, 211):
-            info = info[0]
-            status = 200
-        if status != 200:
-            return
+    def __read_disc_index_async(self, device):
+        """This function must be run async because it does slow I/O"""
+        logger.info('Starting to read disc index')
+
+        if CdPlugin.discid_parser is not None:
+            try:
+                disc_id = CdPlugin.discid_parser.read_disc_id(device)
+                logger.debug(
+                    'Successfully read CD using discid with %i tracks. '
+                    'Musicbrainz id: %s',
+                    len(disc_id.tracks),
+                    disc_id.id,
+                )
+                GLib.idle_add(self.__apply_disc_index, disc_id, None, None)
+                return
+            except Exception:
+                logger.warn('Failed to read from cd using discid.', exc_info=True)
+
+        if sys.platform.startswith('linux'):
+            try:
+                (toc_entries, mcn) = linux_cd_parser.read_cd_index(device)
+                GLib.idle_add(self.__apply_disc_index, None, toc_entries, mcn)
+                return
+            except Exception:
+                logger.warn('Failed to read metadata from CD.', exc_info=True)
+
+        GLib.idle_add(self.__apply_disc_index, None, None, None)
+
+    def __apply_disc_index(self, disc_id, toc_entries, mcn):
+        """This function must be run sync because it accesses the track database"""
+        logger.debug('Applying disc contents to playlist')
+        if disc_id is not None:
+            tracks = CdPlugin.discid_parser.parse_disc(disc_id, self.__device)
+            if tracks is not None:
+                allow_internet = settings.get_option(
+                    'cd_import/fetch_metadata_from_internet', True
+                )
+                if allow_internet:
+                    logger.info('Starting to get disc metadata')
+                    self.__fetch_disc_metadata(disc_id, tracks)
+        elif toc_entries is not None:
+            tracks = linux_cd_parser.parse_tracks(toc_entries, mcn, self.__device)
+        else:
+            logger.error('Could not read disc index')
+            tracks = None
+        if tracks is not None:
+            logger.debug('Read disc with tracks %s', tracks)
+            self.extend(tracks)
+        event.log_event('cd_info_retrieved', self, None)
 
-        (status, info) = CDDB.read(info['category'], info['disc_id'])
-
-        title = info['DTITLE'].split(" / ")
-        for i in range(self.__device[1]):
-            tr = self[i]
-            tr.set_tags(
-                title=info['TTITLE' + str(i)].decode('iso-8859-15', 'replace'),
-                album=title[1].decode('iso-8859-15', 'replace'),
-                artist=title[0].decode('iso-8859-15', 'replace'),
-                year=info['EXTD'].replace("YEAR: ", ""),
-                genre=info['DGENRE'],
+    @common.threaded
+    def __fetch_disc_metadata(self, disc_id, tracks):
+        # TODO: show progress during work
+
+        # TODO: Add more providers?
+        # Discogs:
+        #    Problem: Barely documented, no known support for disc_id
+        # * https://github.com/discogs/discogs_client
+        # * https://www.discogs.com/developers/
+        # CDDB/freedb:
+        # * old python code: http://pycddb.sourceforge.net/
+        # * even older python code: http://cddb-py.sourceforge.net/
+        # * http://ftp.freedb.org/pub/freedb/latest/DBFORMAT
+        # * http://ftp.freedb.org/pub/freedb/latest/CDDBPROTO
+        # * Servers: http://freedb.freedb.org/,
+        if CdPlugin.musicbrainzngs_parser is not None:
+            musicbrainz_data = CdPlugin.musicbrainzngs_parser.fetch_with_disc_id(
+                disc_id
+            )
+            GLib.idle_add(
+                self.__musicbrainz_metadata_fetched, musicbrainz_data, disc_id, tracks
             )
 
-        self.name = title[1].decode('iso-8859-15', 'replace')
-        event.log_event('cddb_info_retrieved', self, True)
+    def __musicbrainz_metadata_fetched(self, musicbrainz_data, disc_id, tracks):
+        metadata = CdPlugin.musicbrainzngs_parser.parse(
+            musicbrainz_data, disc_id, tracks
+        )
+        # TODO: progress: finished
+        if metadata is not None:
+            (tracks, title) = metadata
+            logger.info('Finished getting disc metadata. Disc title: %s', title)
+            event.log_event('cd_info_retrieved', self, title)
+            self.name = title
 
 
 class CDDevice(KeyedDevice):
@@ -223,26 +265,21 @@ def __init__(self, dev):
         self.name = _("Audio Disc")
         self.dev = dev
 
-    def _get_panel_type(self):
-        import imp
-
-        try:
-            _cdguipanel = imp.load_source(
-                "_cdguipanel", os.path.join(os.path.dirname(__file__), "_cdguipanel.py")
-            )
-            return _cdguipanel.CDPanel
-        except Exception:
-            logger.exception("Could not import cd gui panel")
-            return 'flatplaylist'
+    panel_type = _cdguipanel.CDPanel
 
-    panel_type = property(_get_panel_type)
+    def __on_cd_info_retrieved(self, _event_type, cd_playlist, _disc_title):
+        self.playlists.append(cd_playlist)
+        self.connected = True
 
     def connect(self):
-        cdpl = CDPlaylist(device=self.dev)
-        self.playlists.append(cdpl)
-        self.connected = True
+        if self.connected:
+            return
+        event.add_ui_callback(self.__on_cd_info_retrieved, 'cd_info_retrieved')
+        CDPlaylist(device=self.dev)
 
     def disconnect(self):
+        if not self.connected:
+            return
         self.playlists = []
         self.connected = False
         CDDevice.destroy(self)

plugins/cd/_cdguipanel.py

@@ -39,6 +39,12 @@
 logger = logging.getLogger(__name__)
 
 
+# TODO: Improve progress update
+# TODO: Ask for folder where to put the disc instead of blindly copying it
+# TODO: add eject button
+# TODO: allow to directly play from panel
+
+
 class CDImportThread(common.ProgressThread):
     def __init__(self, cd_importer):
         common.ProgressThread.__init__(self)
@@ -78,12 +84,20 @@ def __init__(self, *args):
         device.FlatPlaylistDevicePanel.__init__(self, *args)
         self.__importing = False
 
-        event.add_ui_callback(self._tree_queue_draw, 'cddb_info_retrieved')
+        event.add_ui_callback(self._tree_queue_draw, 'cd_info_retrieved')
 
-    def _tree_queue_draw(self, type, cdplaylist, object=None):
+    def _tree_queue_draw(self, _event_type, cdplaylist, disc_title=None):
         if not hasattr(self.fppanel, 'tree'):
             return
 
+        # TODO: set name to panel. This requires a GUI change in
+        # panel/flatplaylist/FlatPlaylistPanel respectively flatplaylist.ui
+        # Do not use self.name because it breaks the panel after switching to another CD
+        if disc_title is None:
+            self.panel_title = _('Unknown disc')
+        else:
+            self.panel_title = disc_title
+
         if cdplaylist in self.device.playlists:
             logger.info("Calling queue_draw for %s", str(cdplaylist))
             self.fppanel.tree.queue_draw()

plugins/cd/cdprefs.py

@@ -36,7 +36,10 @@
 
 FORMAT_WIDGET = None
 
-# TODO: allow setting cddb server?
+
+class ImportMetadataPreference(widgets.CheckPreference):
+    name = 'cd_import/fetch_metadata_from_internet'
+    default = True
 
 
 class OutputFormatPreference(widgets.ComboPreference):
@@ -65,10 +68,10 @@ def on_check_condition(self):
             return False
 
         curiter = self.condition_widget.get_active_iter()
-        format = self.condition_widget.get_model().get_value(curiter, 0)
-        formatinfo = transcoder.FORMATS[format]
-        if self.format != format:
-            self.format = format
+        tc_format = self.condition_widget.get_model().get_value(curiter, 0)
+        formatinfo = transcoder.FORMATS[tc_format]
+        if self.format != tc_format:
+            self.format = tc_format
             default = formatinfo['default']
 
             if self.default != default: