From 8595bf69a25ca043df07e2b5fa97941242cbf560 Mon Sep 17 00:00:00 2001 From: cosven Date: Thu, 20 Dec 2018 00:53:31 +0800 Subject: [PATCH 1/9] =?UTF-8?q?Model=20=E5=8A=9F=E8=83=BD=E5=A2=9E?= =?UTF-8?q?=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model 支持通过 id + 部分显示字段 创建 --- fuocore/models.py | 84 +++++++++++++++++++++++++++++++++++++-------- tests/test_model.py | 37 ++++++++++++++++---- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/fuocore/models.py b/fuocore/models.py index a21aa35..eba0df1 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -4,7 +4,25 @@ fuocore.models ~~~~~~~~~~~~~~ -This modules defines several music models. +这个模块定义了音乐资源的模型，如歌曲模型： ``SongModel`` , 歌手模型： ``ArtistModel`` 。 +它们都类似这样:: + + class XyzModel(BaseModel): + class Meta: + model_type = ModelType.xyz + fields = ['a', 'b', 'c'] + + @property + def ab(self): + return self.a + self.b + +同时，为了减少实现这些模型时带来的重复代码，这里还实现了： + +- ModelMeta: Model 元类，进行一些黑科技处理：比如解析 Model Meta 类 +- ModelMetadata: Model meta 属性对应的类 +- BaseModel: 基类 + +ModelMetadata, ModelMeta, BaseModel 几个类是互相依赖的。 """ from enum import IntEnum @@ -27,6 +45,7 @@ def __init__(self, model_type=ModelType.dummy.value, provider=None, fields=None, + fields_display=None, allow_get=False, allow_batch=False, **kwargs): @@ -37,34 +56,51 @@ def __init__(self, """ self.model_type = model_type self.provider = provider - self.fields = fields + self.fields = fields or [] + self.fields_display = fields_display or [] self.allow_get = allow_get self.allow_batch = allow_batch for key, value in kwargs.items(): setattr(self, key, value) +class display_property: + """Model 的展示字段的描述器""" + def __init__(self, name): + self.name_real = name + self.name_display = name + '_display' + self.value_display = None + + def __get__(self, instance, _=None): + if instance.use_display: + return getattr(instance, self.name_real) + return self.value_display + + def __set__(self, instance, value): + self.value_display = value + + class ModelMeta(type): def __new__(cls, name, bases, attrs): - # get all meta + # 获取 Model 当前以及父类中的 Meta 信息 + # 如果 Meta 中相同字段的属性，子类的值可以覆盖父类的 _metas = [] for base in bases: base_meta = getattr(base, '_meta', None) if base_meta is not None: _metas.append(base_meta) - - # similar with djang/peewee model meta Meta = attrs.pop('Meta', None) if Meta: _metas.append(Meta) - fields = list() - meta_kv = {} + fields = [] + fields_display = [] + meta_kv = {} # 实例化 ModelMetadata 的 kv 对 for _meta in _metas: - inherited_fields = getattr(_meta, 'fields', []) - fields.extend(inherited_fields) + fields.extend(getattr(_meta, 'fields', [])) + fields_display.extend(getattr(_meta, 'fields_display', [])) for k, v in _meta.__dict__.items(): - if k.startswith('_') or k in ('fields', ): + if k.startswith('_') or k in ('fields', 'fields_display'): continue if k == 'model_type': if ModelType(v) != ModelType.dummy: @@ -80,11 +116,17 @@ def __new__(cls, name, bases, attrs): if provider and ModelType(model_type) != ModelType.dummy: provider.set_model_cls(model_type, klass) fields = list(set(fields)) + fields_display = list(set(fields_display)) + + for field in fields_display: + setattr(klass, field + '_display', display_property(field)) # DEPRECATED attribute _meta + # TODO: remove this in verion 2.3 klass._meta = ModelMetadata(model_type=model_type, provider=provider, fields=fields, + fields_display=fields_display, **meta_kv) klass.source = provider.identifier if provider is not None else None # use meta attribute instead of _meta @@ -92,17 +134,19 @@ def __new__(cls, name, bases, attrs): return klass -class Model(object, metaclass=ModelMeta): +class Model(metaclass=ModelMeta): """base class for data models + Usage:: class User(Model): class Meta: fields = ['name', 'title'] + user = UserModel(name='xxx') assert user.name == 'xxx' user2 = UserModel(user) - assert user2.name = 'xxx' + assert user2.name == 'xxx' """ def __init__(self, obj=None, **kwargs): @@ -118,7 +162,6 @@ class BaseModel(Model): """Base model for music resource. :param identifier: model object identifier, unique in each provider - :param source: model object provider identifier :cvar allow_get: meta var, whether model has a valid get method :cvar allow_list: meta var, whether model has a valid list method @@ -133,12 +176,23 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.use_display = kwargs.get('use_display', True) + def __eq__(self, other): if not isinstance(other, BaseModel): return False return all([other.source == self.source, other.identifier == self.identifier, - other._meta.model_type == self._meta.model_type]) + other.meta.model_type == self.meta.model_type]) + + @classmethod + def create_by_display(cls, identifier, **kwargs): + model = cls(identifier=identifier) + model.use_display = False + for k, v in kwargs.items(): + if k in cls.meta.fields_display: + setattr(model, k + '_display', v) + return model @classmethod def get(cls, identifier): @@ -215,7 +269,7 @@ class Meta: model_type = ModelType.song.value # TODO: 支持低/中/高不同质量的音乐文件 fields = ['album', 'artists', 'lyric', 'comments', 'title', 'url', - 'duration',] + 'duration'] @property def artists_name(self): diff --git a/tests/test_model.py b/tests/test_model.py index 9e4ada7..f9614a1 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,9 +1,10 @@ +from collections import namedtuple from unittest import TestCase -from fuocore.models import Model +from fuocore.models import Model, BaseModel -class FakeProvider(object): +class FakeProvider: identifier = 'fake' name = 'fake' @@ -23,7 +24,7 @@ class Meta: provider = provider song = SongModel() - self.assertEqual(song._meta.provider.name, 'fake') + self.assertEqual(song.meta.provider.name, 'fake') def test_meta_class_inherit(self): class SongModel(Model): @@ -34,7 +35,7 @@ class LastSongModel(SongModel): pass song = LastSongModel() - self.assertEqual(song._meta.model_type, 1) + self.assertEqual(song.meta.model_type, 1) def test_meta_class_inherit_with_override(self): class SongModel(Model): @@ -46,5 +47,29 @@ class Meta: provider = provider song = LastSongModel() - self.assertEqual(song._meta.model_type, 1) - self.assertEqual(song._meta.provider.name, 'fake') + self.assertEqual(song.meta.model_type, 1) + self.assertEqual(song.meta.provider.name, 'fake') + + +class TestBaseModel(TestCase): + def test_display_fields(self): + class SongModel(BaseModel): + class Meta: + fields = ['title', 'album'] + fields_display = ['album_name'] + + @property + def album_name(self): + return self.album.name if self.album else '' + + album_name = 'Minutes-to-Midnight' + song = SongModel.create_by_display(identifier=1, album_name=album_name) + self.assertEqual(song.album_name_display, album_name) + self.assertEqual(song.album_name, '') + + real_album_name = 'Minutes to Midnight' + song.title = 'Leave out all the rest' + Album = namedtuple('Album', ('name', )) + song.album = Album(real_album_name) + song.use_display = False + self.assertEqual(song.album_name, real_album_name) From 70f8668eedf92b5c2e76c0065f34410d22709fdd Mon Sep 17 00:00:00 2001 From: cosven Date: Fri, 28 Dec 2018 23:56:52 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E7=BB=99=20BaseModel=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=20=5F=5Fgetattribute=5F=5F=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/model_display.py | 29 ++++++++++++++++++++++++ fuocore/models.py | 47 +++++++++++++++++++++++++++++++++------ fuocore/netease/models.py | 13 ----------- fuocore/xiami/models.py | 13 ----------- 5 files changed, 73 insertions(+), 33 deletions(-) create mode 100755 examples/model_display.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 078ff67..1e58d87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 2.3a0 (WIP) +- Model 支持 `create_by_display` 工厂函数 + - 给 BaseModel 添加 `__getattribute__` 函数 + ## 2.2b1 (2018-12-02) - QQ 音乐支持显示歌手和专辑详情 - 设计变化： diff --git a/examples/model_display.py b/examples/model_display.py new file mode 100755 index 0000000..fdff53f --- /dev/null +++ b/examples/model_display.py @@ -0,0 +1,29 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging + +from fuocore.netease.models import NSongModel + +logging.basicConfig() +logger = logging.getLogger('fuocore') +logger.setLevel(logging.DEBUG) + + +def test_model_display(): + song = NSongModel.create_by_display( + identifier=254548, + title='成全', + artists_name='刘若英') + assert song.album_name_display == '' + assert song.title_display == '成全' + print(song.url, song.title) + assert song.album_name_display != '' + + +def main(): + test_model_display() + + +if __name__ == '__main__': + main() diff --git a/fuocore/models.py b/fuocore/models.py index eba0df1..07f1cd7 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -24,9 +24,11 @@ def ab(self): ModelMetadata, ModelMeta, BaseModel 几个类是互相依赖的。 """ - +import logging from enum import IntEnum +logger = logging.getLogger(__name__) + class ModelType(IntEnum): dummy = 0 @@ -69,10 +71,10 @@ class display_property: def __init__(self, name): self.name_real = name self.name_display = name + '_display' - self.value_display = None + self.value_display = "" def __get__(self, instance, _=None): - if instance.use_display: + if instance.gotten: return getattr(instance, self.name_real) return self.value_display @@ -176,7 +178,8 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.use_display = kwargs.get('use_display', True) + #: 是否已经调用过 gotten，通常也意味着字段是否都已经初始化 + self.gotten = kwargs.get('gotten', True) def __eq__(self, other): if not isinstance(other, BaseModel): @@ -185,10 +188,39 @@ def __eq__(self, other): other.identifier == self.identifier, other.meta.model_type == self.meta.model_type]) + def __getattribute__(self, name): + """ + 获取 model 某一属性时，如果该属性值为 None 且该属性是 field， + 我们认为这个字段还没有被初始化，这时，我们尝试通过获取 model + 详情来初始化这个字段，于此同时，还会重新给除 identifier + 外的所有 fields 重新赋值。 + """ + cls = type(self) + cls_name = cls.__name__ + value = object.__getattribute__(self, name) + if name in cls.meta.fields and value is None: + if cls.meta.allow_get: + logger.info('Field %s value is None, try to get detail.' % name) + obj = cls.get(self.identifier) + if obj is not None: + for field in cls.meta.fields: + if field in ('identifier', ): + continue + # 这里不能使用 getattr，否则有可能会无限 get + fv = object.__getattribute__(obj, field) + setattr(self, field, fv) + self.gotten = True + else: + logger.warning('Model {} get return None'.format(cls_name)) + else: + logger.warning("Model {} does't allow get".format(cls_name)) + value = object.__getattribute__(self, name) + return value + @classmethod def create_by_display(cls, identifier, **kwargs): model = cls(identifier=identifier) - model.use_display = False + model.gotten = False for k, v in kwargs.items(): if k in cls.meta.fields_display: setattr(model, k + '_display', v) @@ -196,9 +228,9 @@ def create_by_display(cls, identifier, **kwargs): @classmethod def get(cls, identifier): - """Model get method + """获取 model 详情 - Model should return a valid object if the identifier is available. + 这个方法必须尽量初始化所有字段，确保它们的值不是 None。 """ @classmethod @@ -270,6 +302,7 @@ class Meta: # TODO: 支持低/中/高不同质量的音乐文件 fields = ['album', 'artists', 'lyric', 'comments', 'title', 'url', 'duration'] + fields_display = ['title', 'artists_name', 'album_name', 'duration'] @property def artists_name(self): diff --git a/fuocore/netease/models.py b/fuocore/netease/models.py index ce7ca08..40e39d1 100644 --- a/fuocore/netease/models.py +++ b/fuocore/netease/models.py @@ -29,19 +29,6 @@ class Meta: allow_get = True provider = provider - def __getattribute__(self, name): - cls = type(self) - value = object.__getattribute__(self, name) - if name in cls._detail_fields and value is None: - logger.debug('Field %s value is None, get model detail first.' % name) - obj = cls.get(self.identifier) - for field in cls._detail_fields: - setattr(self, field, getattr(obj, field)) - value = object.__getattribute__(self, name) - elif name in cls._detail_fields and not value: - logger.warning('Field %s value is not None, but is %s' % (name, value)) - return value - class NSongModel(SongModel, NBaseModel): @classmethod diff --git a/fuocore/xiami/models.py b/fuocore/xiami/models.py index d0496c9..e76c023 100644 --- a/fuocore/xiami/models.py +++ b/fuocore/xiami/models.py @@ -26,19 +26,6 @@ class Meta: allow_get = True provider = provider - def __getattribute__(self, name): - cls = type(self) - value = object.__getattribute__(self, name) - if name in cls._detail_fields and value is None: - logger.debug('Field %s value is None, get model detail first.' % name) - obj = cls.get(self.identifier) - for field in cls._detail_fields: - setattr(self, field, getattr(obj, field)) - value = object.__getattribute__(self, name) - elif name in cls._detail_fields and not value: - logger.warning('Field %s value is not None, but is %s' % (name, value)) - return value - def _deserialize(data, schema_cls): schema = schema_cls(strict=True) From 8e8d079f6bed94bd9a186ec729c0ad4e0125498d Mon Sep 17 00:00:00 2001 From: cosven Date: Thu, 20 Dec 2018 00:53:31 +0800 Subject: [PATCH 3/9] =?UTF-8?q?Model=20=E5=8A=9F=E8=83=BD=E5=A2=9E?= =?UTF-8?q?=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model 支持通过 id + 部分显示字段 创建 --- fuocore/models.py | 84 +++++++++++++++++++++++++++++++++++++-------- tests/test_model.py | 37 ++++++++++++++++---- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/fuocore/models.py b/fuocore/models.py index a21aa35..eba0df1 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -4,7 +4,25 @@ fuocore.models ~~~~~~~~~~~~~~ -This modules defines several music models. +这个模块定义了音乐资源的模型，如歌曲模型： ``SongModel`` , 歌手模型： ``ArtistModel`` 。 +它们都类似这样:: + + class XyzModel(BaseModel): + class Meta: + model_type = ModelType.xyz + fields = ['a', 'b', 'c'] + + @property + def ab(self): + return self.a + self.b + +同时，为了减少实现这些模型时带来的重复代码，这里还实现了： + +- ModelMeta: Model 元类，进行一些黑科技处理：比如解析 Model Meta 类 +- ModelMetadata: Model meta 属性对应的类 +- BaseModel: 基类 + +ModelMetadata, ModelMeta, BaseModel 几个类是互相依赖的。 """ from enum import IntEnum @@ -27,6 +45,7 @@ def __init__(self, model_type=ModelType.dummy.value, provider=None, fields=None, + fields_display=None, allow_get=False, allow_batch=False, **kwargs): @@ -37,34 +56,51 @@ def __init__(self, """ self.model_type = model_type self.provider = provider - self.fields = fields + self.fields = fields or [] + self.fields_display = fields_display or [] self.allow_get = allow_get self.allow_batch = allow_batch for key, value in kwargs.items(): setattr(self, key, value) +class display_property: + """Model 的展示字段的描述器""" + def __init__(self, name): + self.name_real = name + self.name_display = name + '_display' + self.value_display = None + + def __get__(self, instance, _=None): + if instance.use_display: + return getattr(instance, self.name_real) + return self.value_display + + def __set__(self, instance, value): + self.value_display = value + + class ModelMeta(type): def __new__(cls, name, bases, attrs): - # get all meta + # 获取 Model 当前以及父类中的 Meta 信息 + # 如果 Meta 中相同字段的属性，子类的值可以覆盖父类的 _metas = [] for base in bases: base_meta = getattr(base, '_meta', None) if base_meta is not None: _metas.append(base_meta) - - # similar with djang/peewee model meta Meta = attrs.pop('Meta', None) if Meta: _metas.append(Meta) - fields = list() - meta_kv = {} + fields = [] + fields_display = [] + meta_kv = {} # 实例化 ModelMetadata 的 kv 对 for _meta in _metas: - inherited_fields = getattr(_meta, 'fields', []) - fields.extend(inherited_fields) + fields.extend(getattr(_meta, 'fields', [])) + fields_display.extend(getattr(_meta, 'fields_display', [])) for k, v in _meta.__dict__.items(): - if k.startswith('_') or k in ('fields', ): + if k.startswith('_') or k in ('fields', 'fields_display'): continue if k == 'model_type': if ModelType(v) != ModelType.dummy: @@ -80,11 +116,17 @@ def __new__(cls, name, bases, attrs): if provider and ModelType(model_type) != ModelType.dummy: provider.set_model_cls(model_type, klass) fields = list(set(fields)) + fields_display = list(set(fields_display)) + + for field in fields_display: + setattr(klass, field + '_display', display_property(field)) # DEPRECATED attribute _meta + # TODO: remove this in verion 2.3 klass._meta = ModelMetadata(model_type=model_type, provider=provider, fields=fields, + fields_display=fields_display, **meta_kv) klass.source = provider.identifier if provider is not None else None # use meta attribute instead of _meta @@ -92,17 +134,19 @@ def __new__(cls, name, bases, attrs): return klass -class Model(object, metaclass=ModelMeta): +class Model(metaclass=ModelMeta): """base class for data models + Usage:: class User(Model): class Meta: fields = ['name', 'title'] + user = UserModel(name='xxx') assert user.name == 'xxx' user2 = UserModel(user) - assert user2.name = 'xxx' + assert user2.name == 'xxx' """ def __init__(self, obj=None, **kwargs): @@ -118,7 +162,6 @@ class BaseModel(Model): """Base model for music resource. :param identifier: model object identifier, unique in each provider - :param source: model object provider identifier :cvar allow_get: meta var, whether model has a valid get method :cvar allow_list: meta var, whether model has a valid list method @@ -133,12 +176,23 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.use_display = kwargs.get('use_display', True) + def __eq__(self, other): if not isinstance(other, BaseModel): return False return all([other.source == self.source, other.identifier == self.identifier, - other._meta.model_type == self._meta.model_type]) + other.meta.model_type == self.meta.model_type]) + + @classmethod + def create_by_display(cls, identifier, **kwargs): + model = cls(identifier=identifier) + model.use_display = False + for k, v in kwargs.items(): + if k in cls.meta.fields_display: + setattr(model, k + '_display', v) + return model @classmethod def get(cls, identifier): @@ -215,7 +269,7 @@ class Meta: model_type = ModelType.song.value # TODO: 支持低/中/高不同质量的音乐文件 fields = ['album', 'artists', 'lyric', 'comments', 'title', 'url', - 'duration',] + 'duration'] @property def artists_name(self): diff --git a/tests/test_model.py b/tests/test_model.py index 9e4ada7..f9614a1 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,9 +1,10 @@ +from collections import namedtuple from unittest import TestCase -from fuocore.models import Model +from fuocore.models import Model, BaseModel -class FakeProvider(object): +class FakeProvider: identifier = 'fake' name = 'fake' @@ -23,7 +24,7 @@ class Meta: provider = provider song = SongModel() - self.assertEqual(song._meta.provider.name, 'fake') + self.assertEqual(song.meta.provider.name, 'fake') def test_meta_class_inherit(self): class SongModel(Model): @@ -34,7 +35,7 @@ class LastSongModel(SongModel): pass song = LastSongModel() - self.assertEqual(song._meta.model_type, 1) + self.assertEqual(song.meta.model_type, 1) def test_meta_class_inherit_with_override(self): class SongModel(Model): @@ -46,5 +47,29 @@ class Meta: provider = provider song = LastSongModel() - self.assertEqual(song._meta.model_type, 1) - self.assertEqual(song._meta.provider.name, 'fake') + self.assertEqual(song.meta.model_type, 1) + self.assertEqual(song.meta.provider.name, 'fake') + + +class TestBaseModel(TestCase): + def test_display_fields(self): + class SongModel(BaseModel): + class Meta: + fields = ['title', 'album'] + fields_display = ['album_name'] + + @property + def album_name(self): + return self.album.name if self.album else '' + + album_name = 'Minutes-to-Midnight' + song = SongModel.create_by_display(identifier=1, album_name=album_name) + self.assertEqual(song.album_name_display, album_name) + self.assertEqual(song.album_name, '') + + real_album_name = 'Minutes to Midnight' + song.title = 'Leave out all the rest' + Album = namedtuple('Album', ('name', )) + song.album = Album(real_album_name) + song.use_display = False + self.assertEqual(song.album_name, real_album_name) From 9dddb828db3898afdf84227fe52c4520bbdbb608 Mon Sep 17 00:00:00 2001 From: cosven Date: Fri, 28 Dec 2018 23:56:52 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E7=BB=99=20BaseModel=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=20=5F=5Fgetattribute=5F=5F=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/model_display.py | 29 ++++++++++++++++++++++++ fuocore/models.py | 47 +++++++++++++++++++++++++++++++++------ fuocore/netease/models.py | 13 ----------- fuocore/xiami/models.py | 13 ----------- 5 files changed, 73 insertions(+), 33 deletions(-) create mode 100755 examples/model_display.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ed21b44..91b331e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 2.3a0 (WIP) +- Model 支持 `create_by_display` 工厂函数 + - 给 BaseModel 添加 `__getattribute__` 函数 + ## 2.2 (2018-12-28) - 发一个 2.2 的正式版（经过测试，相对稳定） diff --git a/examples/model_display.py b/examples/model_display.py new file mode 100755 index 0000000..fdff53f --- /dev/null +++ b/examples/model_display.py @@ -0,0 +1,29 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging + +from fuocore.netease.models import NSongModel + +logging.basicConfig() +logger = logging.getLogger('fuocore') +logger.setLevel(logging.DEBUG) + + +def test_model_display(): + song = NSongModel.create_by_display( + identifier=254548, + title='成全', + artists_name='刘若英') + assert song.album_name_display == '' + assert song.title_display == '成全' + print(song.url, song.title) + assert song.album_name_display != '' + + +def main(): + test_model_display() + + +if __name__ == '__main__': + main() diff --git a/fuocore/models.py b/fuocore/models.py index eba0df1..07f1cd7 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -24,9 +24,11 @@ def ab(self): ModelMetadata, ModelMeta, BaseModel 几个类是互相依赖的。 """ - +import logging from enum import IntEnum +logger = logging.getLogger(__name__) + class ModelType(IntEnum): dummy = 0 @@ -69,10 +71,10 @@ class display_property: def __init__(self, name): self.name_real = name self.name_display = name + '_display' - self.value_display = None + self.value_display = "" def __get__(self, instance, _=None): - if instance.use_display: + if instance.gotten: return getattr(instance, self.name_real) return self.value_display @@ -176,7 +178,8 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.use_display = kwargs.get('use_display', True) + #: 是否已经调用过 gotten，通常也意味着字段是否都已经初始化 + self.gotten = kwargs.get('gotten', True) def __eq__(self, other): if not isinstance(other, BaseModel): @@ -185,10 +188,39 @@ def __eq__(self, other): other.identifier == self.identifier, other.meta.model_type == self.meta.model_type]) + def __getattribute__(self, name): + """ + 获取 model 某一属性时，如果该属性值为 None 且该属性是 field， + 我们认为这个字段还没有被初始化，这时，我们尝试通过获取 model + 详情来初始化这个字段，于此同时，还会重新给除 identifier + 外的所有 fields 重新赋值。 + """ + cls = type(self) + cls_name = cls.__name__ + value = object.__getattribute__(self, name) + if name in cls.meta.fields and value is None: + if cls.meta.allow_get: + logger.info('Field %s value is None, try to get detail.' % name) + obj = cls.get(self.identifier) + if obj is not None: + for field in cls.meta.fields: + if field in ('identifier', ): + continue + # 这里不能使用 getattr，否则有可能会无限 get + fv = object.__getattribute__(obj, field) + setattr(self, field, fv) + self.gotten = True + else: + logger.warning('Model {} get return None'.format(cls_name)) + else: + logger.warning("Model {} does't allow get".format(cls_name)) + value = object.__getattribute__(self, name) + return value + @classmethod def create_by_display(cls, identifier, **kwargs): model = cls(identifier=identifier) - model.use_display = False + model.gotten = False for k, v in kwargs.items(): if k in cls.meta.fields_display: setattr(model, k + '_display', v) @@ -196,9 +228,9 @@ def create_by_display(cls, identifier, **kwargs): @classmethod def get(cls, identifier): - """Model get method + """获取 model 详情 - Model should return a valid object if the identifier is available. + 这个方法必须尽量初始化所有字段，确保它们的值不是 None。 """ @classmethod @@ -270,6 +302,7 @@ class Meta: # TODO: 支持低/中/高不同质量的音乐文件 fields = ['album', 'artists', 'lyric', 'comments', 'title', 'url', 'duration'] + fields_display = ['title', 'artists_name', 'album_name', 'duration'] @property def artists_name(self): diff --git a/fuocore/netease/models.py b/fuocore/netease/models.py index ce7ca08..40e39d1 100644 --- a/fuocore/netease/models.py +++ b/fuocore/netease/models.py @@ -29,19 +29,6 @@ class Meta: allow_get = True provider = provider - def __getattribute__(self, name): - cls = type(self) - value = object.__getattribute__(self, name) - if name in cls._detail_fields and value is None: - logger.debug('Field %s value is None, get model detail first.' % name) - obj = cls.get(self.identifier) - for field in cls._detail_fields: - setattr(self, field, getattr(obj, field)) - value = object.__getattribute__(self, name) - elif name in cls._detail_fields and not value: - logger.warning('Field %s value is not None, but is %s' % (name, value)) - return value - class NSongModel(SongModel, NBaseModel): @classmethod diff --git a/fuocore/xiami/models.py b/fuocore/xiami/models.py index d0496c9..e76c023 100644 --- a/fuocore/xiami/models.py +++ b/fuocore/xiami/models.py @@ -26,19 +26,6 @@ class Meta: allow_get = True provider = provider - def __getattribute__(self, name): - cls = type(self) - value = object.__getattribute__(self, name) - if name in cls._detail_fields and value is None: - logger.debug('Field %s value is None, get model detail first.' % name) - obj = cls.get(self.identifier) - for field in cls._detail_fields: - setattr(self, field, getattr(obj, field)) - value = object.__getattribute__(self, name) - elif name in cls._detail_fields and not value: - logger.warning('Field %s value is not None, but is %s' % (name, value)) - return value - def _deserialize(data, schema_cls): schema = schema_cls(strict=True) From 48ca1e67673f76cd6528cc80c90ae2ce002bf967 Mon Sep 17 00:00:00 2001 From: cosven Date: Sat, 29 Dec 2018 00:22:52 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E7=BB=99=20netease=20=E5=92=8C=20xiami=20?= =?UTF-8?q?=E7=9A=84=20model=20=E7=A7=BB=E9=99=A4=20=5Fdetail=5Ffields=20?= =?UTF-8?q?=E7=B1=BB=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 +++- fuocore/models.py | 3 ++- fuocore/netease/models.py | 10 ++++------ fuocore/xiami/models.py | 5 ----- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1441414..858cb3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## 2.3a0 (WIP) - Model 支持 `create_by_display` 工厂函数 - - 给 BaseModel 添加 `__getattribute__` 函数 + - 给 BaseModel 添加 `__getattribute__` 方法 + - 给 NBaseModel 和 XBaseModel 移除 `__getattribute__` 方法和 + `_detail_fields` 类属性 ## 2.2 (2018-12-28) - 发一个 2.2 的正式版（经过测试，相对稳定） diff --git a/fuocore/models.py b/fuocore/models.py index 07f1cd7..91149a3 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -200,7 +200,8 @@ def __getattribute__(self, name): value = object.__getattribute__(self, name) if name in cls.meta.fields and value is None: if cls.meta.allow_get: - logger.info('Field %s value is None, try to get detail.' % name) + logger.info("Model {} {}'s value is None, try to get detail." + .format(repr(self), name)) obj = cls.get(self.identifier) if obj is not None: for field in cls.meta.fields: diff --git a/fuocore/netease/models.py b/fuocore/netease/models.py index 40e39d1..1c431a7 100644 --- a/fuocore/netease/models.py +++ b/fuocore/netease/models.py @@ -22,7 +22,6 @@ class NBaseModel(BaseModel): # FIXME: remove _detail_fields and _api to Meta - _detail_fields = () _api = provider.api class Meta: @@ -113,7 +112,6 @@ def lyric(self, value): class NAlbumModel(AlbumModel, NBaseModel): - _detail_fields = ('cover', 'songs', 'artists', ) @classmethod def get(cls, identifier): @@ -135,7 +133,6 @@ def desc(self, value): class NArtistModel(ArtistModel, NBaseModel): - _detail_fields = ('songs', 'cover') @classmethod def get(cls, identifier): @@ -157,7 +154,6 @@ def desc(self, value): class NPlaylistModel(PlaylistModel, NBaseModel): - _detail_fields = ('songs', ) class Meta: fields = ('uid') @@ -166,6 +162,10 @@ class Meta: def get(cls, identifier): data = cls._api.playlist_detail(identifier) playlist, _ = NeteasePlaylistSchema(strict=True).load(data) + + # 当歌单的描述是空时，desc 的值为 None，这里手动设置为空 + if playlist.desc is None: + playlist.desc = '' return playlist def add(self, song_id, allow_exist=True): @@ -194,8 +194,6 @@ class NSearchModel(SearchModel, NBaseModel): class NUserModel(UserModel, NBaseModel): - _detail_fields = ('playlists', 'fav_playlists') - class Meta: fields = ('cookies', ) diff --git a/fuocore/xiami/models.py b/fuocore/xiami/models.py index e76c023..43ad38e 100644 --- a/fuocore/xiami/models.py +++ b/fuocore/xiami/models.py @@ -18,8 +18,6 @@ class XBaseModel(BaseModel): - # FIXME: remove _detail_fields and _api to Meta - _detail_fields = () _api = provider.api class Meta: @@ -75,7 +73,6 @@ def lyric(self, value): class XAlbumModel(AlbumModel, XBaseModel): - _detail_fields = ('songs', 'artists', 'desc') @classmethod def get(cls, identifier): @@ -86,7 +83,6 @@ def get(cls, identifier): class XArtistModel(ArtistModel, XBaseModel): - _detail_fields = ('cover', 'desc') @classmethod def get(cls, identifier): @@ -112,7 +108,6 @@ def songs(self, value): class XPlaylistModel(PlaylistModel, XBaseModel): - _detail_fields = ('songs', 'desc') class Meta: fields = ('uid', ) From 439a4f02b222d1161a2232e5167b544ca3e3d5e1 Mon Sep 17 00:00:00 2001 From: cosven Date: Sat, 29 Dec 2018 01:53:07 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E7=BB=99=20SongModel=20=E5=8A=A0=E4=B8=8A?= =?UTF-8?q?=20duration=5Fms=20=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fuocore/local/models.py | 13 ------------- fuocore/local/provider.py | 2 +- fuocore/local/schemas.py | 37 ++++++++++++++++++++++++------------- fuocore/models.py | 13 +++++++++++-- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/fuocore/local/models.py b/fuocore/local/models.py index 0c44a8e..751b38b 100644 --- a/fuocore/local/models.py +++ b/fuocore/local/models.py @@ -20,19 +20,6 @@ class Meta: allow_get = True provider = provider - def __getattribute__(self, name): - cls = type(self) - value = object.__getattribute__(self, name) - if name in cls._detail_fields and value is None: - logger.debug('Field %s value is None, get model detail first.' % name) - obj = cls.get(self.identifier) - for field in cls._detail_fields: - setattr(self, field, getattr(obj, field)) - value = object.__getattribute__(self, name) - elif name in cls._detail_fields and not value: - logger.debug('Field %s value is not None, but is %s' % (name, value)) - return value - class LSongModel(SongModel, LBaseModel): diff --git a/fuocore/local/provider.py b/fuocore/local/provider.py index 1d54676..0581c1d 100644 --- a/fuocore/local/provider.py +++ b/fuocore/local/provider.py @@ -173,7 +173,7 @@ def analyze_library(self): except Exception as e: logger.exception('Sort album songs failed.') - if album.artists is not None: + if album.artists: album_artist = album.artists[0] if album_artist.identifier not in self._artists: album_artist_data = {'identifier': album_artist.identifier, diff --git a/fuocore/local/schemas.py b/fuocore/local/schemas.py index 5238dd4..57e9834 100644 --- a/fuocore/local/schemas.py +++ b/fuocore/local/schemas.py @@ -7,17 +7,23 @@ from fuocore.utils import elfhash +DEFAULT_TITLE = DEFAULT_ARTIST_NAME = DEFAULT_ALBUM_NAME = 'Unknown' + + class BaseSchema(Schema): identifier = fields.Field(required=True, missing=None) - desc = fields.Str() + + # 本地歌曲目前都不支持描述和封面，将它们设置为空字符串 + desc = fields.Str(missing='') + cover = fields.Str(missing='') class LocalArtistSchema(BaseSchema): # TODO: 添加一个 alias 字段？ name = fields.Str(required=True) cover = fields.Str() # NOTE: 可能需要单独一个 Schema - songs = fields.List(fields.Nested('LocalSongSchema'), missing=None) - albums = fields.List(fields.Nested('LocalAlbumSchema'), missing=None) + songs = fields.List(fields.Nested('LocalSongSchema'), missing=[]) + albums = fields.List(fields.Nested('LocalAlbumSchema'), missing=[]) @post_load def create_model(self, data): @@ -29,8 +35,8 @@ def create_model(self, data): class LocalAlbumSchema(BaseSchema): name = fields.Str(required=True) img = fields.Str() - songs = fields.List(fields.Nested('LocalSongSchema'), missing=None) - artists = fields.List(fields.Nested(LocalArtistSchema), missing=None) + songs = fields.List(fields.Nested('LocalSongSchema'), missing=[]) + artists = fields.List(fields.Nested(LocalArtistSchema), missing=[]) artists_name = fields.Str() @@ -57,14 +63,18 @@ class LocalSongSchema(BaseSchema): def create_model(self, data): return LSongModel(**data) + class EasyMP3MetadataSongSchema(Schema): """EasyMP3 metadata""" url = fields.Str(required=True) duration = fields.Float(required=True) - title = fields.Str(required=True, missing='Unknown') - artists_name = fields.Str(required=True, load_from='artist', missing='') - album_name = fields.Str(required=True, load_from='album', missing='') - album_artist_name = fields.Str(required=True, load_from='albumartist', missing='') + title = fields.Str(required=True, missing=DEFAULT_TITLE) + artists_name = fields.Str(required=True, load_from='artist', + missing=DEFAULT_ARTIST_NAME) + album_name = fields.Str(required=True, load_from='album', + missing=DEFAULT_ALBUM_NAME) + album_artist_name = fields.Str(required=True, load_from='albumartist', + missing=DEFAULT_ARTIST_NAME) track = fields.Str(load_from='tracknumber') disc = fields.Str(load_from='discnumber') date = fields.Str() @@ -73,17 +83,17 @@ class EasyMP3MetadataSongSchema(Schema): @post_load def create_model(self, data): # NOTE: use {title}-{artists_name}-{album_name} as song identifier - identifier_str = '{} - {} - {} - {}'.format(data['title'], data['artists_name'], data['album_name'], - data['duration']) + identifier_str = '{} - {} - {} - {}'.format( + data['title'], data['artists_name'], data['album_name'], data['duration']) data['identifier'] = str(elfhash(base64.b64encode(bytes(identifier_str, 'utf-8')))) song, _ = LocalSongSchema(strict=True).load(data) - if song.album is None and data['album_name']: + if data['album_name']: album_data = {'name': data['album_name'], 'artists_name': data['album_artist_name']} song.album, _ = LocalAlbumSchema(strict=True).load(album_data) - if song.artists is None and data['artists_name']: + if data['artists_name']: song.artists = [] artist_names = [artist.strip() for artist in re.split(r'[,&]', data['artists_name'])] for artist_name in artist_names: @@ -96,6 +106,7 @@ def create_model(self, data): song.disc = data.get('disc', '1/1') song.track = data.get('track', '1/1') song.date = data.get('date', None) + return song diff --git a/fuocore/models.py b/fuocore/models.py index 91149a3..2478f04 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -24,8 +24,10 @@ def ab(self): ModelMetadata, ModelMeta, BaseModel 几个类是互相依赖的。 """ -import logging + from enum import IntEnum +import logging + logger = logging.getLogger(__name__) @@ -303,7 +305,7 @@ class Meta: # TODO: 支持低/中/高不同质量的音乐文件 fields = ['album', 'artists', 'lyric', 'comments', 'title', 'url', 'duration'] - fields_display = ['title', 'artists_name', 'album_name', 'duration'] + fields_display = ['title', 'artists_name', 'album_name', 'duration_ms'] @property def artists_name(self): @@ -313,6 +315,13 @@ def artists_name(self): def album_name(self): return self.album.name if self.album is not None else '' + @property + def duration_ms(self): + if self.duration is not None: + seconds = self.duration / 1000 + m, s = seconds / 60, seconds % 60 + return '{:02}:{:02}'.format(int(m), int(s)) + @property def filename(self): return '{} - {}.mp3'.format(self.title, self.artists_name) From c388d19cd800bb8ec7aee9aa0c074f6eafd98daf Mon Sep 17 00:00:00 2001 From: cosven Date: Sat, 29 Dec 2018 03:10:36 +0800 Subject: [PATCH 7/9] add fields_no_get field kind --- fuocore/local/models.py | 2 ++ fuocore/models.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/fuocore/local/models.py b/fuocore/local/models.py index 751b38b..22c9a9e 100644 --- a/fuocore/local/models.py +++ b/fuocore/local/models.py @@ -22,6 +22,8 @@ class Meta: class LSongModel(SongModel, LBaseModel): + class Meta: + fields_no_get = ('lyric', ) @classmethod def get(cls, identifier): diff --git a/fuocore/models.py b/fuocore/models.py index 2478f04..0311b4e 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -50,6 +50,7 @@ def __init__(self, provider=None, fields=None, fields_display=None, + fields_no_get=None, allow_get=False, allow_batch=False, **kwargs): @@ -62,6 +63,7 @@ def __init__(self, self.provider = provider self.fields = fields or [] self.fields_display = fields_display or [] + self.fields_no_get = fields_no_get or [] self.allow_get = allow_get self.allow_batch = allow_batch for key, value in kwargs.items(): @@ -97,14 +99,15 @@ def __new__(cls, name, bases, attrs): if Meta: _metas.append(Meta) - fields = [] - fields_display = [] + kind_fields_map = {'fields': [], + 'fields_display': [], + 'fields_no_get': []} meta_kv = {} # 实例化 ModelMetadata 的 kv 对 for _meta in _metas: - fields.extend(getattr(_meta, 'fields', [])) - fields_display.extend(getattr(_meta, 'fields_display', [])) + for kind, fields in kind_fields_map.items(): + fields.extend(getattr(_meta, kind, [])) for k, v in _meta.__dict__.items(): - if k.startswith('_') or k in ('fields', 'fields_display'): + if k.startswith('_') or k in kind_fields_map: continue if k == 'model_type': if ModelType(v) != ModelType.dummy: @@ -119,8 +122,10 @@ def __new__(cls, name, bases, attrs): model_type = meta_kv.pop('model_type', ModelType.dummy.value) if provider and ModelType(model_type) != ModelType.dummy: provider.set_model_cls(model_type, klass) - fields = list(set(fields)) - fields_display = list(set(fields_display)) + + fields_all = list(set(kind_fields_map['fields'])) + fields_display = list(set(kind_fields_map['fields_display'])) + fields_no_get = list(set(kind_fields_map['fields_no_get'])) for field in fields_display: setattr(klass, field + '_display', display_property(field)) @@ -129,8 +134,9 @@ def __new__(cls, name, bases, attrs): # TODO: remove this in verion 2.3 klass._meta = ModelMetadata(model_type=model_type, provider=provider, - fields=fields, + fields=fields_all, fields_display=fields_display, + fields_no_get=fields_no_get, **meta_kv) klass.source = provider.identifier if provider is not None else None # use meta attribute instead of _meta @@ -175,8 +181,16 @@ class Meta: allow_get = True allow_list = False model_type = ModelType.dummy.value + + #: Model 所有字段，子类可以通过设置该字段以添加其它字段 fields = ['identifier'] + #: Model 用来展示的字段 + fields_display = [] + + #: 不触发 get 的 Model 字段 + fields_no_get = [] + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -192,15 +206,17 @@ def __eq__(self, other): def __getattribute__(self, name): """ - 获取 model 某一属性时，如果该属性值为 None 且该属性是 field， - 我们认为这个字段还没有被初始化，这时，我们尝试通过获取 model + 获取 model 某一属性时，如果该属性值为 None 且该属性是 field + 且该属性允许触发 get 方法，这时，我们尝试通过获取 model 详情来初始化这个字段，于此同时，还会重新给除 identifier - 外的所有 fields 重新赋值。 + 外的所 fields 重新赋值。 """ cls = type(self) cls_name = cls.__name__ value = object.__getattribute__(self, name) - if name in cls.meta.fields and value is None: + if name in cls.meta.fields \ + and name not in cls.meta.fields_no_get \ + and value is None: if cls.meta.allow_get: logger.info("Model {} {}'s value is None, try to get detail." .format(repr(self), name)) From 4f191505db87aeb635249bc04aa2074525ada275 Mon Sep 17 00:00:00 2001 From: cosven Date: Sat, 29 Dec 2018 04:06:46 +0800 Subject: [PATCH 8/9] refactor --- fuocore/local/provider.py | 236 +++++++++++++++++++------------------- fuocore/local/schemas.py | 107 ++--------------- 2 files changed, 129 insertions(+), 214 deletions(-) diff --git a/fuocore/local/provider.py b/fuocore/local/provider.py index 0581c1d..825e08e 100644 --- a/fuocore/local/provider.py +++ b/fuocore/local/provider.py @@ -5,17 +5,19 @@ 这些小部分理论都可以从中拆除。 """ +import base64 import logging import pickle import os +import re from fuzzywuzzy import process from marshmallow.exceptions import ValidationError from mutagen import MutagenError from mutagen.mp3 import EasyMP3 from mutagen.easymp4 import EasyMP4 - from fuocore.provider import AbstractProvider +from fuocore.utils import elfhash from fuocore.utils import log_exectime @@ -41,7 +43,29 @@ def scan_directory(directory, exts=None, depth=2): return media_files -def create_song(fpath): +def gen_id(s): + return str(elfhash(base64.b64encode(bytes(s, 'utf-8')))) + + +def create_artist(identifier, name): + return LArtistModel(identifier=identifier, + name=name, + songs=[], + albums=[], + desc='', + cover='',) + + +def create_album(identifier, name): + return LArtistModel(identifier=identifier, + name=name, + songs=[], + artists=[], + desc='', + cover='',) + + +def add_song(fpath, g_songs, g_artists, g_albums): """ parse music file metadata with Easymp3 and return a song model. @@ -55,7 +79,6 @@ def create_song(fpath): logger.exception('Mutagen parse metadata failed, ignore.') return None - schema = EasyMP3MetadataSongSchema(strict=True) metadata_dict = dict(metadata) for key in metadata.keys(): metadata_dict[key] = metadata_dict[key][0] @@ -66,29 +89,102 @@ def create_song(fpath): url=fpath, duration=metadata.info.length * 1000 # milesecond )) + schema = EasyMP3MetadataSongSchema(strict=True) try: - song, _ = schema.load(metadata_dict) + data, _ = schema.load(metadata_dict) except ValidationError: logger.exeception('解析音乐文件({}) 元数据失败'.format(fpath)) - return song - - -class Scanner: - """本地歌曲扫描器""" - + return + + # NOTE: use {title}-{artists_name}-{album_name} as song identifier + title = data['title'] + album_name = data['album_name'] + artist_name_list = [ + name.strip() + for name in re.split(r'[,&]', data['artists_name'])] + artists_name = ','.join(artist_name_list) + duration = data['duration'] + album_artist_name = data['album_artist_name'] + + # 生成 song model + song_id_str = ' - '.join([title, artists_name, album_name, duration]) + song_id = gen_id(song_id_str) + if song_id in g_songs: + # 剩下 album, artists, lyric 三个字段没有初始化 + song = LSongModel(identifier=song_id, + title=title, + url=fpath, + duration=duration, + comments=[], + # 下面这些字段不向外暴露 + genre=data['genre'], + cover=data['cover'], + date=data['date'], + desc=data['desc'], + disc=data['disc'], + track=data['track']) + else: + song = g_songs[song_id] + logger.warning('duplicate song: {} {}'.format(song.url, fpath)) + return + + # 生成 album artist model + album_artist_id = gen_id(album_artist_name) + if album_artist_id not in g_artists: + album_artist = create_artist(album_artist_id, album_artist_name) + g_artists[album_artist_id] = album_artist + else: + album_artist = g_artists[album_artist_id] + + # 生成 album model + album_id_str = album_name + album_artist_name + album_id = gen_id(album_id_str) + if album_id not in g_albums: + album = create_album(album_id, album_name) + else: + album = g_albums[album_id] + + # 处理专辑的歌手信息，专辑歌手的专辑列表信息 + if album not in album_artist.albums: + album_artist.albums.append(album) + if album_artist not in album.artists: + album.artists.append(album_artist) + + # 处理歌曲的歌手和专辑信息，以及歌手的歌曲列表 + song.artists.append(album_artist) + song.album = album + for artist_name in artist_name_list: + artist_id = gen_id(artist_name) + if artist_id in g_artists: + artist = g_artists[artist_id] + else: + artist = create_artist(identifier=artist_id, name=artist_name) + if artist not in song.artists: + song.artists.append(artist) + if song not in artist.songs: + artist.songs.append(song) + + +class Library: DEFAULT_MUSIC_FOLDER = os.path.expanduser('~') + '/Music' def __init__(self, paths=None, depth=2): - self._songs = [] + self._songs = {} + self._albums = {} + self._artists = {} + self.depth = depth - self.paths = paths or [Scanner.DEFAULT_MUSIC_FOLDER] + self.paths = paths or [Library.DEFAULT_MUSIC_FOLDER] - @property - def songs(self): - return self._songs + def list_songs(self): + return list(self._songs.values()) + + # TODO: + def get_song(self, identifier): + return self._songs.get(identifier) @log_exectime - def run(self): + def scan(self): """scan media files in all paths """ song_exts = ['mp3', 'ogg', 'wma', 'm4a'] @@ -99,90 +195,17 @@ def run(self): logger.debug('正在扫描目录(%s)...', directory) media_files.extend(scan_directory(directory, exts, depth)) - self._songs = [] for fpath in media_files: - song = create_song(fpath) - if song is not None: - self._songs.append(song) - else: - logger.warning('%s can not be recognized', fpath) + add_song(fpath, self._songs, self._artists, self._albums) logger.debug('扫描到 %d 首歌曲', len(self._songs)) - -class DataBase: - def __init__(self): - #: identifier song map: {id: song, ...} - self._songs = dict() - - #: identifier album map: {id: album, ...} - self._albums = dict() - - #: identifier artist map: {id: artist, ...} - self._artists = dict() - - @property - def songs(self): - return self._songs.values() - - @property - def albums(self): - return self._albums.values() - - @property - def artists(self): - return self._artists.values() - - def run(self, songs): - self._songs.clear() - self._albums.clear() - self._artists.clear() - - self.setup_library(songs) - self.analyze_library() - - def setup_library(self, scanner_songs): - for song in scanner_songs: - if song.identifier in self._songs: - continue - self._songs[song.identifier] = song - - if song.album is not None: - album = song.album - if album.identifier not in self._albums: - album_data = {'identifier': album.identifier, - 'name': album.name, - 'artists_name': album.artists[0].name if album.artists else '', - 'songs': []} - self._albums[album.identifier], _ = LocalAlbumSchema(strict=True).load(album_data) - self._albums[album.identifier].songs.append(song) - - if song.artists is not None: - for artist in song.artists: - if artist.identifier not in self._artists: - artist_data = {'identifier': artist.identifier, - 'name': artist.name, - 'songs': [], - 'albums': []} - self._artists[artist.identifier], _ = LocalArtistSchema(strict=True).load(artist_data) - self._artists[artist.identifier].songs.append(song) - - def analyze_library(self): - for album in self._albums.values(): + def sortout(self): + for album in self.albums.values(): try: album.songs.sort(key=lambda x: (int(x.disc.split('/')[0]), int(x.track.split('/')[0]))) except Exception as e: logger.exception('Sort album songs failed.') - if album.artists: - album_artist = album.artists[0] - if album_artist.identifier not in self._artists: - album_artist_data = {'identifier': album_artist.identifier, - 'name': album_artist.name, - 'songs': [], - 'albums': []} - self._artists[album_artist.identifier], _ = LocalArtistSchema(strict=True).load(album_artist_data) - self._artists[album_artist.identifier].albums.append(album) - for artist in self._artists.values(): if artist.albums: artist.albums.sort(key=lambda x: (x.songs[0].date is None, x.songs[0].date), reverse=True) @@ -195,19 +218,11 @@ class LocalProvider(AbstractProvider): def __init__(self): super().__init__() - self.library = DataBase() - self._songs = [] - self._albums = [] - self._artists = [] + self.library = Library() def scan(self, paths=None, depth=3): - scanner = Scanner(paths or [], depth=depth) - scanner.run() - - self.library.run(scanner.songs) - self._songs = list(self.library.songs) - self._albums = list(self.library.albums) - self._artists = list(self.library.artists) + self.library.scan() + self.library.sortout() @property def identifier(self): @@ -219,15 +234,8 @@ def name(self): @property def songs(self): - return self._songs - - @property - def artists(self): - return self._artists - - @property - def albums(self): - return self._albums + # DEPRECATED + return self.library.list_songs() @log_exectime def search(self, keyword, **kwargs): @@ -248,7 +256,5 @@ def search(self, keyword, **kwargs): provider = LocalProvider() -from .schemas import LocalAlbumSchema -from .schemas import LocalArtistSchema from .schemas import EasyMP3MetadataSongSchema -from .models import LSearchModel +from .models import LSearchModel, LSongModel, LAlbumModel, LArtistModel diff --git a/fuocore/local/schemas.py b/fuocore/local/schemas.py index 57e9834..dc88bf4 100644 --- a/fuocore/local/schemas.py +++ b/fuocore/local/schemas.py @@ -1,69 +1,10 @@ # -*- coding: utf-8 -*- -import re -import base64 - -from marshmallow import Schema, fields, post_load - -from fuocore.utils import elfhash +from marshmallow import Schema, fields DEFAULT_TITLE = DEFAULT_ARTIST_NAME = DEFAULT_ALBUM_NAME = 'Unknown' -class BaseSchema(Schema): - identifier = fields.Field(required=True, missing=None) - - # 本地歌曲目前都不支持描述和封面，将它们设置为空字符串 - desc = fields.Str(missing='') - cover = fields.Str(missing='') - - -class LocalArtistSchema(BaseSchema): - # TODO: 添加一个 alias 字段？ - name = fields.Str(required=True) - cover = fields.Str() # NOTE: 可能需要单独一个 Schema - songs = fields.List(fields.Nested('LocalSongSchema'), missing=[]) - albums = fields.List(fields.Nested('LocalAlbumSchema'), missing=[]) - - @post_load - def create_model(self, data): - if data['identifier'] is None: - data['identifier'] = str(elfhash(base64.b64encode(bytes(data['name'], 'utf-8')))) - return LArtistModel(**data) - - -class LocalAlbumSchema(BaseSchema): - name = fields.Str(required=True) - img = fields.Str() - songs = fields.List(fields.Nested('LocalSongSchema'), missing=[]) - artists = fields.List(fields.Nested(LocalArtistSchema), missing=[]) - - artists_name = fields.Str() - - @post_load - def create_model(self, data): - if data['identifier'] is None: - identifier_str = '{} - {}'.format(data['name'], data['artists_name']) - data['identifier'] = str(elfhash(base64.b64encode(bytes(identifier_str, 'utf-8')))) - album = LAlbumModel(**data) - if album.artists is None and data['artists_name']: - album_artist, _ = LocalArtistSchema(strict=True).load({'name': data['artists_name']}) - album.artists = [album_artist] - return album - - -class LocalSongSchema(BaseSchema): - title = fields.Str(required=True) - url = fields.Str(required=True) - duration = fields.Float(required=True) # mileseconds - album = fields.Nested(LocalAlbumSchema, missing=None) - artists = fields.List(fields.Nested(LocalArtistSchema), missing=None) - - @post_load - def create_model(self, data): - return LSongModel(**data) - - class EasyMP3MetadataSongSchema(Schema): """EasyMP3 metadata""" url = fields.Str(required=True) @@ -75,43 +16,11 @@ class EasyMP3MetadataSongSchema(Schema): missing=DEFAULT_ALBUM_NAME) album_artist_name = fields.Str(required=True, load_from='albumartist', missing=DEFAULT_ARTIST_NAME) - track = fields.Str(load_from='tracknumber') - disc = fields.Str(load_from='discnumber') - date = fields.Str() - genre = fields.Str() - - @post_load - def create_model(self, data): - # NOTE: use {title}-{artists_name}-{album_name} as song identifier - identifier_str = '{} - {} - {} - {}'.format( - data['title'], data['artists_name'], data['album_name'], data['duration']) - data['identifier'] = str(elfhash(base64.b64encode(bytes(identifier_str, 'utf-8')))) - song, _ = LocalSongSchema(strict=True).load(data) - - if data['album_name']: - album_data = {'name': data['album_name'], - 'artists_name': data['album_artist_name']} - song.album, _ = LocalAlbumSchema(strict=True).load(album_data) + track = fields.Str(load_from='tracknumber', missing='1/1') + disc = fields.Str(load_from='discnumber', missing='1/1') + date = fields.Str(missing='') + genre = fields.Str(missing='') - if data['artists_name']: - song.artists = [] - artist_names = [artist.strip() for artist in re.split(r'[,&]', data['artists_name'])] - for artist_name in artist_names: - artist_data = {'name': artist_name} - artist, _ = LocalArtistSchema(strict=True).load(artist_data) - song.artists.append(artist) - - song.genre = data.get('genre', None) - if song.album is not None: - song.disc = data.get('disc', '1/1') - song.track = data.get('track', '1/1') - song.date = data.get('date', None) - - return song - - -from .models import ( - LAlbumModel, - LArtistModel, - LSongModel, -) + # 本地歌曲目前都不支持描述和封面，将它们设置为空字符串 + desc = fields.Str(missing='') + cover = fields.Str(missing='') From 94fc6e4894e527e55890641bd8c7eee683251149 Mon Sep 17 00:00:00 2001 From: cosven Date: Sat, 29 Dec 2018 11:37:51 +0800 Subject: [PATCH 9/9] bugfix --- fuocore/local/models.py | 7 ++-- fuocore/local/provider.py | 66 ++++++++++++++++++++++---------------- fuocore/models.py | 19 +++++++---- fuocore/netease/models.py | 6 +--- fuocore/netease/schemas.py | 2 +- 5 files changed, 58 insertions(+), 42 deletions(-) diff --git a/fuocore/local/models.py b/fuocore/local/models.py index 22c9a9e..256a830 100644 --- a/fuocore/local/models.py +++ b/fuocore/local/models.py @@ -23,11 +23,12 @@ class Meta: class LSongModel(SongModel, LBaseModel): class Meta: + fields = ('disc', 'genre', 'date', 'track', 'cover', 'desc') fields_no_get = ('lyric', ) @classmethod def get(cls, identifier): - return cls.meta.provider.library._songs.get(identifier) + return cls.meta.provider.library.get_song(identifier) @classmethod def list(cls, identifier_list): @@ -39,7 +40,7 @@ class LAlbumModel(AlbumModel, LBaseModel): @classmethod def get(cls, identifier): - return cls.meta.provider.library._albums.get(identifier) + return cls.meta.provider.library.get_album(identifier) class LArtistModel(ArtistModel, LBaseModel): @@ -47,7 +48,7 @@ class LArtistModel(ArtistModel, LBaseModel): @classmethod def get(cls, identifier): - return cls.meta.provider.library._artists.get(identifier) + return cls.meta.provider.library.get_artist(identifier) class LSearchModel(SearchModel, LBaseModel): diff --git a/fuocore/local/provider.py b/fuocore/local/provider.py index 825e08e..3c61279 100644 --- a/fuocore/local/provider.py +++ b/fuocore/local/provider.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +# pylint: disable=wrong-import-position """ TODO: 这个模块中目前逻辑非常多，包括音乐目录扫描、音乐库的构建等小部分， 这些小部分理论都可以从中拆除。 @@ -7,7 +7,6 @@ import base64 import logging -import pickle import os import re @@ -57,12 +56,12 @@ def create_artist(identifier, name): def create_album(identifier, name): - return LArtistModel(identifier=identifier, - name=name, - songs=[], - artists=[], - desc='', - cover='',) + return LAlbumModel(identifier=identifier, + name=name, + songs=[], + artists=[], + desc='', + cover='',) def add_song(fpath, g_songs, g_artists, g_albums): @@ -107,11 +106,12 @@ def add_song(fpath, g_songs, g_artists, g_albums): album_artist_name = data['album_artist_name'] # 生成 song model - song_id_str = ' - '.join([title, artists_name, album_name, duration]) + song_id_str = ' - '.join([title, artists_name, album_name, str(int(duration))]) song_id = gen_id(song_id_str) - if song_id in g_songs: - # 剩下 album, artists, lyric 三个字段没有初始化 + if song_id not in g_songs: + # 剩下 album, lyric 三个字段没有初始化 song = LSongModel(identifier=song_id, + artists=[], title=title, url=fpath, duration=duration, @@ -123,9 +123,10 @@ def add_song(fpath, g_songs, g_artists, g_albums): desc=data['desc'], disc=data['disc'], track=data['track']) + g_songs[song_id] = song else: song = g_songs[song_id] - logger.warning('duplicate song: {} {}'.format(song.url, fpath)) + logger.warning('duplicate song: %s %s', song.url, fpath) return # 生成 album artist model @@ -141,17 +142,19 @@ def add_song(fpath, g_songs, g_artists, g_albums): album_id = gen_id(album_id_str) if album_id not in g_albums: album = create_album(album_id, album_name) + g_albums[album_id] = album else: album = g_albums[album_id] - # 处理专辑的歌手信息，专辑歌手的专辑列表信息 + # 处理专辑的歌手信息和歌曲信息，专辑歌手的专辑列表信息 if album not in album_artist.albums: album_artist.albums.append(album) if album_artist not in album.artists: album.artists.append(album_artist) + if song not in album.songs: + album.songs.append(song) # 处理歌曲的歌手和专辑信息，以及歌手的歌曲列表 - song.artists.append(album_artist) song.album = album for artist_name in artist_name_list: artist_id = gen_id(artist_name) @@ -159,6 +162,7 @@ def add_song(fpath, g_songs, g_artists, g_albums): artist = g_artists[artist_id] else: artist = create_artist(identifier=artist_id, name=artist_name) + g_artists[artist_id] = artist if artist not in song.artists: song.artists.append(artist) if song not in artist.songs: @@ -168,39 +172,43 @@ def add_song(fpath, g_songs, g_artists, g_albums): class Library: DEFAULT_MUSIC_FOLDER = os.path.expanduser('~') + '/Music' - def __init__(self, paths=None, depth=2): + def __init__(self): self._songs = {} self._albums = {} self._artists = {} - self.depth = depth - self.paths = paths or [Library.DEFAULT_MUSIC_FOLDER] - def list_songs(self): return list(self._songs.values()) - # TODO: def get_song(self, identifier): return self._songs.get(identifier) + def get_album(self, identifier): + return self._albums.get(identifier) + + def get_artist(self, identifier): + return self._artists.get(identifier) + @log_exectime - def scan(self): + def scan(self, paths=None, depth=2): """scan media files in all paths """ song_exts = ['mp3', 'ogg', 'wma', 'm4a'] exts = song_exts - depth = self.depth if self.depth <= 3 else 3 + paths = paths or [Library.DEFAULT_MUSIC_FOLDER] + depth = depth if depth <= 3 else 3 media_files = [] - for directory in self.paths: + for directory in paths: logger.debug('正在扫描目录(%s)...', directory) media_files.extend(scan_directory(directory, exts, depth)) + logger.info('共扫描到 %d 个音乐文件，准备将其录入本地音乐库', len(media_files)) for fpath in media_files: add_song(fpath, self._songs, self._artists, self._albums) - logger.debug('扫描到 %d 首歌曲', len(self._songs)) + logger.info('录入本地音乐库完毕') def sortout(self): - for album in self.albums.values(): + for album in self._albums.values(): try: album.songs.sort(key=lambda x: (int(x.disc.split('/')[0]), int(x.track.split('/')[0]))) except Exception as e: @@ -221,7 +229,7 @@ def __init__(self): self.library = Library() def scan(self, paths=None, depth=3): - self.library.scan() + self.library.scan(paths, depth) self.library.sortout() @property @@ -234,7 +242,6 @@ def name(self): @property def songs(self): - # DEPRECATED return self.library.list_songs() @log_exectime @@ -257,4 +264,9 @@ def search(self, keyword, **kwargs): provider = LocalProvider() from .schemas import EasyMP3MetadataSongSchema -from .models import LSearchModel, LSongModel, LAlbumModel, LArtistModel +from .models import ( + LSearchModel, + LSongModel, + LAlbumModel, + LArtistModel, +) diff --git a/fuocore/models.py b/fuocore/models.py index 0311b4e..24292f0 100644 --- a/fuocore/models.py +++ b/fuocore/models.py @@ -44,6 +44,13 @@ class ModelType(IntEnum): user = 17 +class ModelStage(IntEnum): + """Model 所处的阶段，有大小关系""" + display = 4 + inited = 8 + gotten = 16 + + class ModelMetadata(object): def __init__(self, model_type=ModelType.dummy.value, @@ -78,7 +85,7 @@ def __init__(self, name): self.value_display = "" def __get__(self, instance, _=None): - if instance.gotten: + if instance.stage >= ModelStage.inited: return getattr(instance, self.name_real) return self.value_display @@ -194,8 +201,7 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - #: 是否已经调用过 gotten，通常也意味着字段是否都已经初始化 - self.gotten = kwargs.get('gotten', True) + self.stage = kwargs.get('stage', ModelStage.inited) def __eq__(self, other): if not isinstance(other, BaseModel): @@ -216,7 +222,8 @@ def __getattribute__(self, name): value = object.__getattribute__(self, name) if name in cls.meta.fields \ and name not in cls.meta.fields_no_get \ - and value is None: + and value is None \ + and self.stage < ModelStage.gotten: if cls.meta.allow_get: logger.info("Model {} {}'s value is None, try to get detail." .format(repr(self), name)) @@ -228,7 +235,7 @@ def __getattribute__(self, name): # 这里不能使用 getattr，否则有可能会无限 get fv = object.__getattribute__(obj, field) setattr(self, field, fv) - self.gotten = True + self.stage = ModelStage.gotten else: logger.warning('Model {} get return None'.format(cls_name)) else: @@ -239,7 +246,7 @@ def __getattribute__(self, name): @classmethod def create_by_display(cls, identifier, **kwargs): model = cls(identifier=identifier) - model.gotten = False + model.stage = ModelStage.display for k, v in kwargs.items(): if k in cls.meta.fields_display: setattr(model, k + '_display', v) diff --git a/fuocore/netease/models.py b/fuocore/netease/models.py index 1c431a7..0ea6172 100644 --- a/fuocore/netease/models.py +++ b/fuocore/netease/models.py @@ -154,7 +154,6 @@ def desc(self, value): class NPlaylistModel(PlaylistModel, NBaseModel): - class Meta: fields = ('uid') @@ -162,10 +161,6 @@ class Meta: def get(cls, identifier): data = cls._api.playlist_detail(identifier) playlist, _ = NeteasePlaylistSchema(strict=True).load(data) - - # 当歌单的描述是空时，desc 的值为 None，这里手动设置为空 - if playlist.desc is None: - playlist.desc = '' return playlist def add(self, song_id, allow_exist=True): @@ -196,6 +191,7 @@ class NSearchModel(SearchModel, NBaseModel): class NUserModel(UserModel, NBaseModel): class Meta: fields = ('cookies', ) + fields_no_get = ('cookies', ) @classmethod def get(cls, identifier): diff --git a/fuocore/netease/schemas.py b/fuocore/netease/schemas.py index 5a8a0e1..c3ae263 100644 --- a/fuocore/netease/schemas.py +++ b/fuocore/netease/schemas.py @@ -76,7 +76,7 @@ def create_model(self, data): song.album.cover = None song.album.songs = None if data.get('desc') is None: - data.pop('desc') + data['desc'] = '' return NPlaylistModel(**data)