feat: custom fields can be set by plugins for all library items

This is probably the most egregiously disgusting commit, so for that, I apologize. It turns out that creating custom fields required or prompted *a lot* of code changes in order to integrate properly. Thus, summing this commit up as a single "feature" seems laughable, but I couldn't be bothered to properly split it up. I'll try to summarize all the main changes though:

1. Use json in sqlite to allow any plugin to set custom fields. These fields can be queried, edited, accessed, etc. transparently from other fields.
2. Musicbrainz ids were moved to custom fields. This is because I want core fields to solely be anything that is fundamental to a track, album, or extra. Custom fields are for any plugin-specific fields, like in this case, musicbrainz ids.
3. Add custom duplicate detection for items in the library. Because we previously used musibrainz's release-ids for duplicate detection, I added the ability for any plugin to add constraints to whether or not a library item was considered "unique" from another.
4. New duplicate detection and resolution. Duplicate detection, now based off of core and custom constraints, happens automatically whenever an item is changed or added to the library. This functionality, along with the existing duplicate prompt, was moved into a separate 'duplicate' plugin.

I'm sure there's even more that happened here that I forgot about.

Moving musicbrainz ids to custom fields prompted a full database schema rewrite rather than a migration, so this commit is not compatible with any before it. I'm hoping this will be the last database breaking change, and that we can rely on migrations going forward to ensure backwards compatability from this point on.

alembic/versions/530acc53e20c_initial_generation.py → alembic/versions/0aae92be930b_initial_generation.py

@@ -1,17 +1,18 @@
 """initial generation.
 
-Revision ID: 530acc53e20c
+Revision ID: 0aae92be930b
 Revises: 
-Create Date: 2022-08-17 16:54:33.820645
+Create Date: 2022-09-11 09:32:40.094676
 
 """
 import sqlalchemy as sa
+from sqlalchemy.ext.mutable import MutableDict
 
 import moe
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = "530acc53e20c"
+revision = "0aae92be930b"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -25,22 +26,34 @@ def upgrade():
         sa.Column("artist", sa.String(), nullable=False),
         sa.Column("date", sa.Date(), nullable=False),
         sa.Column("disc_total", sa.Integer(), nullable=False, default=1),
-        sa.Column("mb_album_id", sa.String(), nullable=True),
         sa.Column("path", moe.library.lib_item.PathType(), nullable=False),
         sa.Column("title", sa.String(), nullable=False),
+        sa.Column(
+            "_custom_fields",
+            MutableDict.as_mutable(sa.JSON(none_as_null=True)),
+            nullable=False,
+            default={},
+        ),
         sa.PrimaryKeyConstraint("_id"),
-        sa.UniqueConstraint("mb_album_id"),
         sa.UniqueConstraint("path"),
     )
     op.create_table(
         "genre",
+        sa.Column("_id", sa.Integer(), nullable=False),
         sa.Column("name", sa.String(), nullable=False),
-        sa.PrimaryKeyConstraint("name"),
+        sa.PrimaryKeyConstraint("_id"),
+        sa.UniqueConstraint("name", sqlite_on_conflict="REPLACE"),
     )
     op.create_table(
-        "extras",
+        "extra",
         sa.Column("_id", sa.Integer(), nullable=False),
         sa.Column("path", moe.library.lib_item.PathType(), nullable=False),
+        sa.Column(
+            "_custom_fields",
+            MutableDict.as_mutable(sa.JSON(none_as_null=True)),
+            nullable=False,
+            default="{}",
+        ),
         sa.Column("_album_id", sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(
             ["_album_id"],
@@ -52,32 +65,36 @@ def upgrade():
     op.create_table(
         "track",
         sa.Column("_id", sa.Integer(), nullable=False),
-        sa.Column("artist", sa.String(), nullable=True),
-        sa.Column("disc", sa.Integer(), nullable=False, default=1),
-        sa.Column("mb_track_id", sa.String(), nullable=True),
+        sa.Column("artist", sa.String(), nullable=False),
+        sa.Column("disc", sa.Integer(), nullable=False),
         sa.Column("path", moe.library.lib_item.PathType(), nullable=False),
-        sa.Column("title", sa.String(), nullable=True),
+        sa.Column("title", sa.String(), nullable=False),
         sa.Column("track_num", sa.Integer(), nullable=False),
+        sa.Column(
+            "_custom_fields",
+            MutableDict.as_mutable(sa.JSON(none_as_null=True)),
+            nullable=False,
+            default="{}",
+        ),
         sa.Column("_album_id", sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(
             ["_album_id"],
             ["album._id"],
         ),
         sa.PrimaryKeyConstraint("_id"),
         sa.UniqueConstraint("disc", "track_num", "_album_id"),
-        sa.UniqueConstraint("mb_track_id"),
         sa.UniqueConstraint("path"),
     )
     op.create_table(
         "track_genre",
         sa.Column("genre", sa.String(), nullable=True),
-        sa.Column("track_id", sa.Integer(), nullable=True),
+        sa.Column("track", sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(
             ["genre"],
-            ["genre.name"],
+            ["genre._id"],
         ),
         sa.ForeignKeyConstraint(
-            ["track_id"],
+            ["track"],
             ["track._id"],
         ),
     )
@@ -88,7 +105,7 @@ def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table("track_genre")
     op.drop_table("track")
-    op.drop_table("extras")
+    op.drop_table("extra")
     op.drop_table("genre")
     op.drop_table("album")
     # ### end Alembic commands ###

docs/developers/api/core.rst

@@ -24,9 +24,21 @@ Library
 
 ``library`` subpackage.
 
+General
+-------
+
+``lib_item`` submodule.
+
+.. automodule:: moe.library.lib_item
+   :members:
+   :exclude-members: cache_ok
+
+
 Album
 -----
 
+``album`` submodule.
+
 .. automodule:: moe.library.album
    :members:
    :exclude-members: year, path
@@ -49,18 +61,9 @@ Extra
 
 .. automodule:: moe.library.extra
    :members:
-   :exclude-members: filename, path
+   :exclude-members: path
    :show-inheritance:
 
-Library Item
-------------
-
-``lib_item`` submodule.
-
-.. automodule:: moe.library.lib_item
-   :members:
-   :exclude-members: cache_ok
-
 
 Query
 =====

docs/developers/api/hooks.core.rst

@@ -10,6 +10,32 @@ Config
 .. autoclass:: moe.config.Hooks
    :members:
 
+Library
+=======
+
+General
+-------
+.. autoclass:: moe.library.lib_item.Hooks
+   :members:
+
+Albums
+------
+
+.. autoclass:: moe.library.album.Hooks
+   :members:
+
+Tracks
+------
+
+.. autoclass:: moe.library.track.Hooks
+   :members:
+
+Extras
+------
+
+.. autoclass:: moe.library.extra.Hooks
+   :members:
+
 
 Plugins
 =======
@@ -23,8 +49,3 @@ Import
 ------
 .. autoclass:: moe.plugins.moe_import.import_core.Hooks
    :members:
-
-Remove
-------
-.. autoclass:: moe.plugins.remove.rm_core.Hooks
-   :members:

docs/fields.rst

@@ -12,7 +12,8 @@ Track Fields
 ************
 .. csv-table::
     :header: "Field", "Description", "Notes"
-    :widths: 20, 50, 50
+    :widths: 4, 10, 6
+    :width: 100%
 
     "album", "Album title", ""
     "albumartist", "Album artist", ""
@@ -21,8 +22,6 @@ Track Fields
     "disc", "Disc number", ""
     "disc_total", "Number of discs in the album", ""
     "genre", "Genre", "Supports multiple values"
-    "mb_album_id", "Musicbrainz album release ID", ""
-    "mb_track_id", "Musicbrainz release track ID", ""
     "path", "Filesystem path of the track", ""
     "title", "Track title", ""
     "track_num", "Track number", ""
@@ -33,12 +32,12 @@ Album Fields
 ************
 .. csv-table::
     :header: "Field", "Description", "Notes"
-    :widths: 20, 50, 50
+    :widths: 4, 10, 6
+    :width: 100%
 
     "artist", "Album artist", ""
     "date", "Album release date", "YYYY-MM-DD format"
     "disc_total", "Number of discs in the album", ""
-    "mb_album_id", "Musicbrainz album release ID", ""
     "path", "Filesystem path of the album", ""
     "title", "Album title", ""
     "year", "Album release year", ""
@@ -48,7 +47,12 @@ Extra Fields
 ************
 .. csv-table::
     :header: "Field", "Description", "Notes"
-    :widths: 20, 50, 50
+    :widths: 4, 10, 6
+    :width: 100%
 
-    "filename", "The filename of the extra.", ""
     "path", "Filesystem path of the extra", ""
+
+*************
+Custom Fields
+*************
+In addition to the above fields, plugins may add any number of custom fields to Moe. These fields don't behave any differently i.e. they can be queried, edited, accessed, etc, the same as any normal field. You can check each plugin's documentation for more information on the custom fields they expose.

docs/plugins/add.rst

@@ -3,9 +3,6 @@ Add
 ###
 Adds music to your library.
 
-.. note::
-    In order to add a track to your library, Moe requires each track to contain, at a minimum, a track number, an albumartist (or artist), and a date (or year).
-
 *************
 Configuration
 *************

docs/plugins/duplicate.rst

@@ -0,0 +1,11 @@
+#########
+Duplicate
+#########
+Handles detection and resolution of duplicate items in the library.
+
+Duplicate items will automatically be detected upon any library additions or changes, and the user will be asked how they'd like to resolve the duplicates.
+
+*************
+Configuration
+*************
+The ``duplicate`` plugin is enabled by default.

docs/plugins/move.rst

@@ -30,7 +30,7 @@ Path Configuration Options
         - The ``if`` statement inside the path simply means that if there is more than one disc in the album, the tracks will be put into separate directories for their respective disc.
         - Include ``track.path.suffix`` at the end if you wish to retain the file extension of the track file.
 
-``extra_path = "{extra.filename}"``
+``extra_path = "{extra.path.name}"``
     Extra filesystem path format relative to ``album_path``.
 
 Paths are formatted using python `f-strings <https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals>`_ which, as demonstrated by the default track path, allow all the advanced formatting and expression evaluation that come with them. You can access any of the :ref:`respective item's fields <fields:Fields>` in these strings using ``{[album/track/extra].field}`` notation as shown.

docs/plugins/musicbrainz.rst

@@ -38,3 +38,25 @@ The following options involve auto updating a specific collection on musicbrainz
 
 ``auto_remove = False``
     Whether to automatically remove releases from ``collection_id`` when removed from the library.
+
+*************
+Custom Fields
+*************
+
+Track Fields
+------------
+.. csv-table::
+    :header: "Field", "Description", "Notes"
+    :widths: 4, 10, 6
+    :width: 100%
+
+    "mb_track_id", "Musicbrainz track id", ""
+
+Album Fields
+------------
+.. csv-table::
+    :header: "Field", "Description", "Notes"
+    :widths: 4, 10, 6
+    :width: 100%
+
+    "mb_album_id", "Musicbrainz album aka release id", ""

docs/plugins/plugins.rst

@@ -11,6 +11,7 @@ These are all the plugins that are enabled by default.
    :maxdepth: 1
 
    add
+   duplicate
    edit
    import
    info

docs/query.rst

@@ -3,9 +3,9 @@ Querying
 ########
 Many plugins use a "query" to search for music in your library.
 
-The query must be in the format ``field:value`` where field is a :ref:`track's field <fields:Track Fields>` to match and value is that field's value. Internally, this ``field:value`` pair is referred to as a single "term". The match is case-insensitive.
+The query must be in the format ``field:value`` where ``field`` is, by default, a :ref:`track's field <fields:Track Fields>` to match and ``value`` is the field's value (case-insensitive). To match an :ref:`album's field <fields:Album Fields>` or an :ref:`extra's field <fields:Extra Fields>`, prepend the field with ``a:`` or ``e:`` respectively. Internally, this ``field:value`` pair is referred to as a single "term".
 
-Album queries, specified with the ``-a, --album`` option, will return albums that contain any tracks matching the given query. Similarly, querying for extras, specified with the ``-e, --extra`` option, will return extras that are attached to albums that contain any tracks matching the given query.
+By default, tracks will be returned by the query, but you can choose to return albums by using the ``-a, --album`` option, or you can return extras using the ``-e, --extra`` option.
 
 If you would like to specify a value with whitespace or multiple words, enclose the
 term in quotes.
@@ -49,9 +49,13 @@ For example, to match all Wu-Tang Clan tracks that start with the letter 'A', us
 .. note::
     When using multiple terms, they are joined together using AND logic, meaning all terms must be true to return a match.
 
+.. tip::
+    Fields of different types can be mixed and matched in a query string. For example, the query ``--extras 'a:album:The College Dropout' e:path:%jpg$`` will return any extras with the 'jpg' file extension belonging to the album titled 'The College Dropout'.
+
 .. tip::
     Normal queries may be faster when compared to regular expression queries. If you are experiencing performance issues with regex queries, see if you can make an equivalent normal query using the LIKE wildcard characters.
 
+
 ****************
 Supported Fields
 ****************

moe/cli.py

@@ -61,9 +61,13 @@ def add_command(cmd_parsers: argparse._SubParsersAction):
                     parents=[moe.cli.query_parser],
                 )
 
+            Then, you can call ``query.query(args.query, query_type=args.query_type)``
+            to get a list of items matching the query from the library.
+
         See Also:
-            `The python documentation for adding sub-parsers.
-            <https://docs.python.org/3/library/argparse.html#sub-commands>`_
+            * `The python documentation for adding sub-parsers.
+              <https://docs.python.org/3/library/argparse.html#sub-commands>`_
+            * The :meth:`~moe.query.query` function.
         """