From 80beecce5cb9b208479309e176090285c9c4a1fa Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 23 Mar 2020 18:01:40 +0100 Subject: [PATCH 1/3] removed forgotten print command --- beetsplug/xtractor/command.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/beetsplug/xtractor/command.py b/beetsplug/xtractor/command.py index df9b2bf..cddf5db 100644 --- a/beetsplug/xtractor/command.py +++ b/beetsplug/xtractor/command.py @@ -242,8 +242,6 @@ def _run_analysis_high_level(self, item): self._say("Attribute not present: {0}".format(e)) return - print(audiodata) - if not self.cfg_dry_run: for attr in audiodata.keys(): if audiodata.get(attr): From f2294cc1c50cf71607f989f82c687631b009e276 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 23 Mar 2020 23:11:39 +0100 Subject: [PATCH 2/3] config controlled query and externally callable command --- BEETSDIR/config.yaml | 11 +-- beetsplug/xtractor/command.py | 98 ++++++++++++--------------- beetsplug/xtractor/config_default.yml | 66 ++++++++++++++---- beetsplug/xtractor/version.py | 2 +- 4 files changed, 102 insertions(+), 75 deletions(-) diff --git a/BEETSDIR/config.yaml b/BEETSDIR/config.yaml index 05a8b0d..a4819fd 100644 --- a/BEETSDIR/config.yaml +++ b/BEETSDIR/config.yaml @@ -2,8 +2,11 @@ # USAGE: export environment variable BEETSDIR=/path/to/BEETSDIR # then: `beets config -p` should list this file -directory: Music -library: library.db +directory: /Volumes/J/Music/ +library: ~/.config/beets/real_library.db +#directory: Music +#library: library.db + asciify_paths: yes id3v23: yes @@ -27,14 +30,14 @@ xtractor: threads: 1 force: no quiet: no - items_per_run: 0 + items_per_run: 1 keep_output: yes keep_profiles: no output_path: /Users/jackisback/Documents/Projects/Python/BeetsPluginXtractor/BEETSDIR/xtraction low_level_extractor: /Users/jackisback/Documents/Projects/Other/extractors/beta5/essentia_streaming_extractor_music high_level_extractor: /Users/jackisback/Documents/Projects/Other/extractors/beta5/essentia_streaming_extractor_music_svm low_level_profile: - outputFormat: yaml + outputFormat: json outputFrames: 0 high_level_profile: outputFormat: json diff --git a/beetsplug/xtractor/command.py b/beetsplug/xtractor/command.py index cddf5db..0594375 100644 --- a/beetsplug/xtractor/command.py +++ b/beetsplug/xtractor/command.py @@ -14,6 +14,7 @@ import yaml from beets import dbcore +from beets.dbcore import types from beets.library import Library, Item, parse_query_string from beets.ui import Subcommand, decargs from confuse import Subview @@ -36,6 +37,8 @@ class XtractorCommand(Subcommand): query = None parser = None + items_to_analyse = None + cfg_auto = False cfg_dry_run = False cfg_write = True @@ -58,7 +61,7 @@ def __init__(self, config): self.cfg_quiet = cfg.get("quiet") self.cfg_items_per_run = cfg.get("items_per_run") - self.parser = OptionParser(usage='%prog [options] [QUERY...]') + self.parser = OptionParser(usage='beet xtractor [options] [QUERY...]') self.parser.add_option( '-d', '--dry-run', @@ -133,68 +136,56 @@ def func(self, lib: Library, options, arguments): self.xtract() def xtract(self): + self.find_items_to_analyse() + self._say("Number of items to be processed: {}".format(len(self.items_to_analyse))) + + # Count only and exit + if self.cfg_count_only: + return + + # Limit the number of items per run (0 means no limit) + if self.cfg_items_per_run != 0: + self.items_to_analyse = self.items_to_analyse[:self.cfg_items_per_run] + self._say("Number of items selected: {}".format(len(self.items_to_analyse))) + + # Run tasks on selected items + self._execute_on_each_items(self.items_to_analyse, self.run_full_analysis) + + # Delete profiles (if config wants) + if self.config["keep_profiles"].exists() and not self.config["keep_profiles"].get(): + os.unlink(self._get_extractor_profile_path("low")) + os.unlink(self._get_extractor_profile_path("high")) + + def find_items_to_analyse(self): # Parse the incoming query parsed_query, parsed_sort = parse_query_string(" ".join(self.query), Item) combined_query = parsed_query - # Add unprocessed items query = "bpm:0 , gender::^$" + # Add unprocessed items query if not self.cfg_force: # Set up the query for unprocessed items - unprocessed_items_query = dbcore.query.OrQuery( - [ - # LOW - # dbcore.query.NoneQuery(u'average_loudness', fast=False), - dbcore.query.MatchQuery(u'average_loudness', None, fast=False), - dbcore.query.NumericQuery(u'bpm', u'0'), - dbcore.query.MatchQuery(u'danceability', None, fast=False), - dbcore.query.MatchQuery(u'beats_count', None, fast=False), - - # HIGH - dbcore.query.MatchQuery(u'danceable', None, fast=False), - dbcore.query.MatchQuery(u'gender', None, fast=False), - dbcore.query.MatchQuery(u'genre_rosamerica', None, fast=False), - dbcore.query.MatchQuery(u'voice_instrumental', None, fast=False), - - dbcore.query.MatchQuery(u'mood_acoustic', None, fast=False), - dbcore.query.MatchQuery(u'mood_aggressive', None, fast=False), - dbcore.query.MatchQuery(u'mood_electronic', None, fast=False), - dbcore.query.MatchQuery(u'mood_happy', None, fast=False), - dbcore.query.MatchQuery(u'mood_party', None, fast=False), - dbcore.query.MatchQuery(u'mood_relaxed', None, fast=False), - dbcore.query.MatchQuery(u'mood_sad', None, fast=False), - ] - ) + subqueries = [] + target_maps = ["low_level_targets", "high_level_targets"] + for map_key in target_maps: + target_map = self.config[map_key] + for fld in target_map: + if target_map[fld]["required"].exists() and target_map[fld]["required"].get(bool): + fast = fld in Item._fields + query_item = dbcore.query.MatchQuery(fld, None, fast=fast) + subqueries.append(query_item) + + unprocessed_items_query = dbcore.query.OrQuery(subqueries) combined_query = dbcore.query.AndQuery([parsed_query, unprocessed_items_query]) log.debug("Combined query: {}".format(combined_query)) # Get the library items - library_items = self.lib.items(combined_query, parsed_sort) - if len(library_items) == 0: + self.items_to_analyse = self.lib.items(combined_query, parsed_sort) + if len(self.items_to_analyse) == 0: self._say("No items to process") return - # Count only and exit - if self.cfg_count_only: - self._say("Number of items to be processed: {}".format(len(library_items))) - return - - # Limit the number of items per run (0 means no limit) - items = [] - for item in library_items: - items.append(item) - if self.cfg_items_per_run != 0 and len(items) >= self.cfg_items_per_run: - break - - self._say("Number of items selected: {}".format(len(items))) - self._execute_on_each_items(items, self._run_full_analysis) - - # Delete profiles (if config wants) - if self.config["keep_profiles"].exists() and not self.config["keep_profiles"].get(): - os.unlink(self._get_extractor_profile_path("low")) - os.unlink(self._get_extractor_profile_path("high")) - - def _run_full_analysis(self, item): + def run_full_analysis(self, item): self._run_analysis_low_level(item) self._run_analysis_high_level(item) self._run_write_to_item(item) @@ -235,12 +226,10 @@ def _run_analysis_high_level(self, item): try: target_map = self.config["high_level_targets"] audiodata = bpmHelper.extract_from_output(output_path, target_map) + log.debug("Audiodata(High): {}".format(audiodata)) except FileNotFoundError as e: self._say("File not found: {0}".format(e)) return - except KeyError as e: - self._say("Attribute not present: {0}".format(e)) - return if not self.cfg_dry_run: for attr in audiodata.keys(): @@ -270,13 +259,10 @@ def _run_analysis_low_level(self, item): try: target_map = self.config["low_level_targets"] audiodata = bpmHelper.extract_from_output(output_path, target_map) - + log.debug("Audiodata(Low): {}".format(audiodata)) except FileNotFoundError as e: self._say("File not found: {0}".format(e)) return - except AttributeError as e: - self._say("Attribute not present: {0}".format(e)) - return if not self.cfg_dry_run: for attr in audiodata.keys(): diff --git a/beetsplug/xtractor/config_default.yml b/beetsplug/xtractor/config_default.yml index 5687f13..1a4b7fe 100644 --- a/beetsplug/xtractor/config_default.yml +++ b/beetsplug/xtractor/config_default.yml @@ -4,15 +4,19 @@ write: yes threads: 1 force: no quiet: no +items_per_run: 0 +keep_output: no +keep_profiles: no low_level_targets: - average_loudness: - path: "lowlevel.average_loudness" - type: float - bpm: - path: "rhythm.bpm" - type: integer - danceability: - path: "rhythm.danceability" + average_loudness: + path: "lowlevel.average_loudness" + type: float + bpm: + path: "rhythm.bpm" + type: integer + required: yes + danceability: + path: "rhythm.danceability" type: float beats_count: path: "rhythm.beats_count" @@ -21,9 +25,11 @@ high_level_targets: danceable: path: "highlevel.danceability.all.danceable" type: float + required: yes gender: path: "highlevel.gender.value" type: string + required: yes is_male: path: "highlevel.gender.all.male" type: float @@ -33,9 +39,11 @@ high_level_targets: genre_rosamerica: path: "highlevel.genre_rosamerica.value" type: string + required: yes voice_instrumental: path: "highlevel.voice_instrumental.value" type: string + required: yes is_voice: path: "highlevel.voice_instrumental.all.voice" type: float @@ -45,21 +53,51 @@ high_level_targets: mood_acoustic: path: "highlevel.mood_acoustic.all.acoustic" type: float + required: yes mood_aggressive: path: "highlevel.mood_aggressive.all.aggressive" type: float + required: yes mood_electronic: path: "highlevel.mood_electronic.all.electronic" type: float + required: yes mood_happy: path: "highlevel.mood_happy.all.happy" type: float + required: yes mood_party: - path: "highlevel.mood_party.all.party" - type: float + path: "highlevel.mood_party.all.party" + type: float + required: yes mood_relaxed: - path: "highlevel.mood_relaxed.all.relaxed" - type: float + path: "highlevel.mood_relaxed.all.relaxed" + type: float + required: yes mood_sad: - path: "highlevel.mood_sad.all.sad" - type: float + path: "highlevel.mood_sad.all.sad" + type: float + required: yes +low_level_extractor: /usr/lib/extractors/beta5/essentia_streaming_extractor_music +high_level_extractor: /usr/lib/extractors/beta5/essentia_streaming_extractor_music_svm +low_level_profile: + outputFormat: json + outputFrames: 0 +high_level_profile: + outputFormat: json + highlevel: + compute: 1 + svm_models: + - /usr/lib/extractors/svm_models_beta5/danceability.history + - /usr/lib/extractors/svm_models_beta5/gender.history + - /usr/lib/extractors/svm_models_beta5/genre_rosamerica.history + - /usr/lib/extractors/svm_models_beta5/mood_acoustic.history + - /usr/lib/extractors/svm_models_beta5/mood_aggressive.history + - /usr/lib/extractors/svm_models_beta5/mood_electronic.history + - /usr/lib/extractors/svm_models_beta5/mood_happy.history + - /usr/lib/extractors/svm_models_beta5/mood_party.history + - /usr/lib/extractors/svm_models_beta5/mood_relaxed.history + - /usr/lib/extractors/svm_models_beta5/mood_sad.history + - /usr/lib/extractors/svm_models_beta5/voice_instrumental.history +chromaprint: + compute: 0 \ No newline at end of file diff --git a/beetsplug/xtractor/version.py b/beetsplug/xtractor/version.py index 12381f8..ae88e4f 100644 --- a/beetsplug/xtractor/version.py +++ b/beetsplug/xtractor/version.py @@ -4,4 +4,4 @@ # Created: 3/13/20, 12:17 AM # License: See LICENSE.txt -__version__ = '0.2.0' +__version__ = '0.2.1' From 28ab1092fc21abdb229efbdb1577af4724cde056 Mon Sep 17 00:00:00 2001 From: Adam Jakab Date: Mon, 23 Mar 2020 23:17:29 +0100 Subject: [PATCH 3/3] default config indentation fix --- beetsplug/xtractor/config_default.yml | 120 +++++++++++++------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/beetsplug/xtractor/config_default.yml b/beetsplug/xtractor/config_default.yml index 1a4b7fe..f39f3c1 100644 --- a/beetsplug/xtractor/config_default.yml +++ b/beetsplug/xtractor/config_default.yml @@ -17,67 +17,67 @@ low_level_targets: required: yes danceability: path: "rhythm.danceability" - type: float - beats_count: - path: "rhythm.beats_count" - type: integer + type: float + beats_count: + path: "rhythm.beats_count" + type: integer high_level_targets: - danceable: - path: "highlevel.danceability.all.danceable" - type: float - required: yes - gender: - path: "highlevel.gender.value" - type: string - required: yes - is_male: - path: "highlevel.gender.all.male" - type: float - is_female: - path: "highlevel.gender.all.female" - type: float - genre_rosamerica: - path: "highlevel.genre_rosamerica.value" - type: string - required: yes - voice_instrumental: - path: "highlevel.voice_instrumental.value" - type: string - required: yes - is_voice: - path: "highlevel.voice_instrumental.all.voice" - type: float - is_instrumental: - path: "highlevel.voice_instrumental.all.instrumental" - type: float - mood_acoustic: - path: "highlevel.mood_acoustic.all.acoustic" - type: float - required: yes - mood_aggressive: - path: "highlevel.mood_aggressive.all.aggressive" - type: float - required: yes - mood_electronic: - path: "highlevel.mood_electronic.all.electronic" - type: float - required: yes - mood_happy: - path: "highlevel.mood_happy.all.happy" - type: float - required: yes - mood_party: - path: "highlevel.mood_party.all.party" - type: float - required: yes - mood_relaxed: - path: "highlevel.mood_relaxed.all.relaxed" - type: float - required: yes - mood_sad: - path: "highlevel.mood_sad.all.sad" - type: float - required: yes + danceable: + path: "highlevel.danceability.all.danceable" + type: float + required: yes + gender: + path: "highlevel.gender.value" + type: string + required: yes + is_male: + path: "highlevel.gender.all.male" + type: float + is_female: + path: "highlevel.gender.all.female" + type: float + genre_rosamerica: + path: "highlevel.genre_rosamerica.value" + type: string + required: yes + voice_instrumental: + path: "highlevel.voice_instrumental.value" + type: string + required: yes + is_voice: + path: "highlevel.voice_instrumental.all.voice" + type: float + is_instrumental: + path: "highlevel.voice_instrumental.all.instrumental" + type: float + mood_acoustic: + path: "highlevel.mood_acoustic.all.acoustic" + type: float + required: yes + mood_aggressive: + path: "highlevel.mood_aggressive.all.aggressive" + type: float + required: yes + mood_electronic: + path: "highlevel.mood_electronic.all.electronic" + type: float + required: yes + mood_happy: + path: "highlevel.mood_happy.all.happy" + type: float + required: yes + mood_party: + path: "highlevel.mood_party.all.party" + type: float + required: yes + mood_relaxed: + path: "highlevel.mood_relaxed.all.relaxed" + type: float + required: yes + mood_sad: + path: "highlevel.mood_sad.all.sad" + type: float + required: yes low_level_extractor: /usr/lib/extractors/beta5/essentia_streaming_extractor_music high_level_extractor: /usr/lib/extractors/beta5/essentia_streaming_extractor_music_svm low_level_profile: