Merge pull request #20 from JackTheMico/develop

fix(class): Remove pytest mark from main logic;Fix filters logic

ruia_peewee_async/__init__.py

@@ -7,7 +7,6 @@
 from typing import Optional as TOptional
 from typing import Sequence, Tuple, Union
 
-import pytest
 from peewee import DoesNotExist, Model, Query
 from peewee_async import (
     AsyncQueryWrapper,
@@ -88,27 +87,15 @@ def _check_result(data: Tuple):
 result_validator = Schema(Use(_check_result))
 
 
-async def filter_func(data, spider_ins, database, manager, model, filters) -> bool:
-    if not hasattr(spider_ins, f"{database}_filters"):
-        conditions = [getattr(model, fil) for fil in filters]
-        filter_res = await manager.execute(model.select(*conditions).distinct())
-        setattr(spider_ins, f"{database}_filters", filter_res)
-    filtered = False
-    filter_res = getattr(spider_ins, f"{database}_filters")
-    for fil in filters:
-        fil_res = [getattr(x, fil) for x in filter_res]
-        if not fil_res:
-            continue
-        outfil = data[fil]
-        if not isinstance(outfil, type(fil_res[0])):
-            outfil = (
-                filter_res[0]  # pylint: disable=protected-access
-                ._meta.columns[fil]
-                .adapt(outfil)
-            )
-        if outfil in fil_res:
-            filtered = True
-    return filtered
+async def filter_func(data, manager, model, filters) -> bool:
+    conditions = [getattr(model, fil) for fil in filters]
+    query = {x.name: data[x.name] for x in conditions}
+    try:
+        await manager.get(model, **query)
+    except DoesNotExist:
+        return False
+    else:
+        return True
 
 
 class RuiaPeeweeInsert:
@@ -154,9 +141,7 @@ async def process(spider_ins: Spider, callback_result):
             manager: Manager = getattr(spider_ins, f"{database}_manager")
             model: Model = getattr(spider_ins, f"{database}_model")
             if filters:
-                filtered = await filter_func(
-                    data, spider_ins, database, manager, model, filters
-                )
+                filtered = await filter_func(data, manager, model, filters)
                 if filtered:
                     msg += (
                         f"<RuiaPeeweeAsync: data: {data} was filtered by filters: {filters},"
@@ -226,9 +211,7 @@ async def _deal_update(
             manager: Manager = getattr(spider_ins, f"{database}_manager")
             model: Model = getattr(spider_ins, f"{database}_model")
             if filters:
-                filtered = await filter_func(
-                    data, spider_ins, database, manager, model, filters
-                )
+                filtered = await filter_func(data, manager, model, filters)
                 if filtered:
                     msg += f"<RuiaPeeweeAsync: data: {data} was filtered by filters: {filters}\n"
                     continue
@@ -421,7 +404,6 @@ async def init_after_start(spider_ins):
     return init_after_start
 
 
-@pytest.mark.no_cover
 async def before_stop(spider_ins):
     if hasattr(spider_ins, "postgres_manager"):
         await spider_ins.postgres_manager.close()

tests/test_both.py

@@ -5,7 +5,7 @@
 import pytest
 from peewee import CharField
 
-from ruia_peewee_async import TargetDB, after_start, create_model
+from ruia_peewee_async import TargetDB, after_start, create_model, before_stop
 
 from .common import Insert, Update
 
@@ -135,8 +135,8 @@ async def test_both_filters(self, mysql, postgresql, event_loop, caplog):
         prows = await spider_ins.postgres_manager.count(
             spider_ins.postgres_model.select()
         )
-        assert mrows == 20
-        assert prows == 20
+        assert mrows == 11
+        assert prows == 11
         spider_ins = await BothUpdate.async_start(
             loop=event_loop,
             after_start=after_start(mysql=mysql, postgres=postgresql),
@@ -150,8 +150,8 @@ async def test_both_filters(self, mysql, postgresql, event_loop, caplog):
         prows = await spider_ins.postgres_manager.count(
             spider_ins.postgres_model.select()
         )
-        assert mrows == 20
-        assert prows == 20
+        assert mrows == 11
+        assert prows == 11
 
     @pytest.mark.dependency(depends=["TestBoth::test_both_not_update_when_exists"])
     async def test_both_update(self, mysql, postgresql, event_loop):
@@ -233,3 +233,21 @@ async def test_both_create_when_not_exists(self, mysql, postgresql, event_loop):
             )
             await asyncio.sleep(1)
         assert mrows_after == 10
+
+    @pytest.mark.dependency(depends=["TestBoth::test_both_create_when_not_exists"])
+    async def test_both_before_stop(self, mysql, postgresql, event_loop, caplog):
+        mysql, postgresql = basic_setup(mysql, postgresql)
+        await BothInsert.async_start(
+            loop=event_loop,
+            after_start=after_start(mysql=mysql, postgres=postgresql),
+            target_db=TargetDB.BOTH,
+            before_stop=before_stop,
+        )
+        await BothUpdate.async_start(
+            loop=event_loop,
+            after_start=after_start(mysql=mysql, postgres=postgresql),
+            target_db=TargetDB.BOTH,
+            before_stop=before_stop,
+        )
+        assert "RuntimeError" not in caplog.text
+        assert "Exception" not in caplog.text

tests/test_mysql.py

@@ -5,7 +5,7 @@
 import pytest
 from peewee import CharField
 
-from ruia_peewee_async import after_start, create_model
+from ruia_peewee_async import after_start, create_model, before_stop
 
 from .common import Insert, Update
 
@@ -96,7 +96,7 @@ async def test_mysql_filters(self, mysql, event_loop, caplog):
         )
         assert "wasn't filtered by filters" in caplog.text
         rows = await spider_ins.mysql_manager.count(spider_ins.mysql_model.select())
-        assert rows == 20
+        assert rows == 11
         spider_ins = await MySQLUpdate.async_start(
             loop=event_loop,
             after_start=after_start(mysql=mysql),
@@ -106,7 +106,7 @@ async def test_mysql_filters(self, mysql, event_loop, caplog):
         )
         assert "was filtered by filters" in caplog.text
         rows = await spider_ins.mysql_manager.count(spider_ins.mysql_model.select())
-        assert rows == 20
+        assert rows == 11
 
     @pytest.mark.dependency(depends=["TestMySQL::test_mysql_not_update_when_exists"])
     async def test_mysql_update(self, mysql, event_loop):
@@ -145,7 +145,6 @@ async def test_mysql_dont_create_when_not_exists(self, mysql, event_loop):
     )
     async def test_mysql_create_when_not_exists(self, mysql, event_loop):
         mysql = basic_setup(mysql)
-        # mysql["model"]["table_name"] = "ruia_mysql_notexist"
         model, _ = create_model(create_table=True, mysql=mysql)
         rows_before = model.select().count()
         assert rows_before == 0
@@ -162,3 +161,19 @@ async def test_mysql_create_when_not_exists(self, mysql, event_loop):
             )
             await asyncio.sleep(1)
         assert rows_after == 10
+
+    @pytest.mark.dependency(depends=["TestMySQL::test_mysql_create_when_not_exists"])
+    async def test_mysql_before_stop(self, mysql, event_loop, caplog):
+        mysql = basic_setup(mysql)
+        await MySQLInsert.async_start(
+            loop=event_loop,
+            after_start=after_start(mysql=mysql),
+            before_stop=before_stop,
+        )
+        await MySQLUpdate.async_start(
+            loop=event_loop,
+            after_start=after_start(mysql=mysql),
+            before_stop=before_stop,
+        )
+        assert "RuntimeError" not in caplog.text
+        assert "Exception" not in caplog.text

tests/test_postgresql.py

@@ -5,7 +5,7 @@
 import pytest
 from peewee import CharField
 
-from ruia_peewee_async import TargetDB, after_start, create_model
+from ruia_peewee_async import TargetDB, after_start, create_model, before_stop
 
 from .common import Insert, Update
 
@@ -109,7 +109,7 @@ async def test_postgres_filters(self, postgresql, event_loop, caplog):
         rows = await spider_ins.postgres_manager.count(
             spider_ins.postgres_model.select()
         )
-        assert rows == 20
+        assert rows == 11
         spider_ins = await PostgresqlUpdate.async_start(
             loop=event_loop,
             after_start=after_start(postgres=postgresql),
@@ -122,7 +122,7 @@ async def test_postgres_filters(self, postgresql, event_loop, caplog):
         rows = await spider_ins.postgres_manager.count(
             spider_ins.postgres_model.select()
         )
-        assert rows == 20
+        assert rows == 11
 
     @pytest.mark.dependency(
         depends=["TestPostgreSQL::test_postgres_not_update_when_exists"]
@@ -166,7 +166,6 @@ async def test_postgres_dont_create_when_not_exists(self, postgresql, event_loop
     )
     async def test_postgres_create_when_not_exists(self, postgresql, event_loop):
         postgresql = basic_setup(postgresql)
-        # postgresql["model"]["table_name"] = "ruia_postgres_notexist"
         model, _ = create_model(create_table=True, postgres=postgresql)
         rows_before = model.select().count()
         assert rows_before == 0
@@ -184,3 +183,23 @@ async def test_postgres_create_when_not_exists(self, postgresql, event_loop):
             )
             await asyncio.sleep(1)
         assert rows_after == 10
+
+    @pytest.mark.dependency(
+        depends=["TestPostgreSQL::test_postgres_create_when_not_exists"]
+    )
+    async def test_postgres_before_stop(self, postgresql, event_loop, caplog):
+        postgresql = basic_setup(postgresql)
+        await PostgresqlInsert.async_start(
+            loop=event_loop,
+            after_start=after_start(postgres=postgresql),
+            target_db=TargetDB.POSTGRES,
+            before_stop=before_stop,
+        )
+        await PostgresqlUpdate.async_start(
+            loop=event_loop,
+            after_start=after_start(postgres=postgresql),
+            target_db=TargetDB.POSTGRES,
+            before_stop=before_stop,
+        )
+        assert "RuntimeError" not in caplog.text
+        assert "Exception" not in caplog.text