style: update code style

shengchenyang · Feb 23, 2024 · 1fc1fcf · 1fc1fcf
1 parent bbacd81
commit 1fc1fcf
Show file tree

Hide file tree

Showing 19 changed files with 116 additions and 94 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,17 +13,17 @@ repos:
     -   id: check-added-large-files
 
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
     -   id: flake8
 
 -   repo: https://github.com/psf/black.git
-    rev: 23.9.1
+    rev: 24.2.0
     hooks:
     -   id: black
 
 -   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
     -   id: isort
 

diff --git a/ayugespidertools/commands/crawl.py b/ayugespidertools/commands/crawl.py
@@ -1,5 +1,4 @@
 from scrapy.commands.crawl import Command
 
 
-class AyuCommand(Command):
-    ...
+class AyuCommand(Command): ...
diff --git a/ayugespidertools/common/expend.py b/ayugespidertools/common/expend.py
@@ -64,7 +64,9 @@ def _connect(
         except Exception as e:
             # (1049, "Unknown database 'xxx'")
             if "1049" in str(e):
-                logger.warning(f"目标数据库：{mysql_conf.database} 不存在，尝试创建中...")
+                logger.warning(
+                    f"目标数据库：{mysql_conf.database} 不存在，尝试创建中..."
+                )
                 # 如果连接目标数据库报不存在的错误时，先创建出此目标数据库
                 ReuseOperation.create_database(db_conf=mysql_conf)
             else:
@@ -132,7 +134,9 @@ def _get_log_by_spider(self, spider, crawl_time):
                         if status_code == "429":
                             error_reason += f"{status_code}错误：代理超过使用频率限制"
                         else:
-                            error_reason += f"{status_code}错误：网页失效/无此网页/网站拒绝访问"
+                            error_reason += (
+                                f"{status_code}错误：网页失效/无此网页/网站拒绝访问"
+                            )
                     elif status_code.startswith("5"):
                         error_reason += f"{status_code}错误：网站服务器处理出错"
                     elif status_code != "":
@@ -209,7 +213,9 @@ def _connect(
         except Exception as e:
             # err: connection to server at "x.x.x.x", port x failed: FATAL:  database "x" does not exist
             if "failed" in str(e).lower():
-                logger.warning(f"目标数据库：{postgres_conf.database} 不存在，尝试创建中...")
+                logger.warning(
+                    f"目标数据库：{postgres_conf.database} 不存在，尝试创建中..."
+                )
                 ReuseOperation.create_database(db_conf=postgres_conf)
             else:
                 logger.error(f"connect to postgresql failed: {e}")

diff --git a/ayugespidertools/common/mongodbpipe.py b/ayugespidertools/common/mongodbpipe.py
@@ -28,6 +28,7 @@ def _get_insert_data(self, item_dict: dict) -> Tuple[dict, str]:
 
         Returns:
             insert_data: 返回 dict 格式的存储数据
+            table_name: item_dict 对应的 table
         """
         insert_data = ReuseOperation.get_items_except_keys(
             dict_conf=item_dict,

diff --git a/ayugespidertools/common/multiplexing.py b/ayugespidertools/common/multiplexing.py
@@ -383,7 +383,9 @@ def create_database(cls, db_conf: Union[MysqlConf, PostgreSQLConf]) -> None:
 
         else:
             assert False, f"Invalid db_conf type: {type(db_conf)}"
-        logger.info(f"创建数据库 {db_conf.database} 成功，其 charset 类型是：{db_conf.charset}!")
+        logger.info(
+            f"创建数据库 {db_conf.database} 成功，其 charset 类型是：{db_conf.charset}!"
+        )
 
     @classmethod
     def dict_keys_to_lower(cls, deal_dict: dict) -> dict:

diff --git a/ayugespidertools/common/sqlformat.py b/ayugespidertools/common/sqlformat.py
@@ -33,8 +33,8 @@ def select_generate(
             limit: limit 限制，默认无限制（查询所有）；如果需要则指定 int 值即可
 
         Returns:
-            select_sql: 生成的 sql 语句
-            tuple(rule.values()): 查询字段的参数名称
+            1). sql: 生成的 sql 语句
+            2). 查询字段的参数名称
         """
         select_key = ", ".join(f"`{k}`" for k in key)
         select_key = select_key.replace("""`count(*)`""", "count(*)")
@@ -48,10 +48,8 @@ def select_generate(
         _where = f"where {select_where}" if select_where else ""
         _order_by = f"order by {order_by}" if order_by else ""
         _limit = f"limit {limit}" if limit else ""
-        select_sql = (
-            f"""select {select_key} from {db_table} {_where} {_order_by} {_limit}"""
-        )
-        return select_sql, tuple(rule.values())
+        sql = f"""select {select_key} from {db_table} {_where} {_order_by} {_limit}"""
+        return sql, tuple(rule.values())
 
     @staticmethod
     def insert_generate(db_table: str, data: dict) -> Tuple[str, tuple]:
@@ -62,8 +60,8 @@ def insert_generate(db_table: str, data: dict) -> Tuple[str, tuple]:
             data: 需要插入的关键字段，key: 数据表字段；value: 需插入的参数名
 
         Returns:
-            select_sql: 生成的 sql 语句
-            tuple(rule.values()): 新增字段的参数名称
+            1). sql: 生成的 sql 语句
+            2). 新增字段的参数名称
         """
         keys = ", ".join(f"`{k}`" for k in data)
         values = ", ".join(["%s"] * len(data))
@@ -83,8 +81,8 @@ def update_generate(
             base: 在有多个查询规则时，选择 "and" 或 "or"，默认 "and"
 
         Returns:
-            select_sql: 生成的 sql 语句
-            tuple(rule.values()): 更新字段的参数名称
+            1). sql: 生成的 sql 语句
+            2). 更新字段的参数名称
         """
         update_set = ", ".join(f"`{k}`=%s" for k in data)
 

diff --git a/ayugespidertools/formatdata.py b/ayugespidertools/formatdata.py
@@ -36,7 +36,7 @@ def click_point_deal(decimal: float, decimal_places: int = 2) -> float:
             decimal_places: 需要保留的小数点位数
 
         Returns:
-            decimal(float): 四舍五入后的小数点
+            1). 四舍五入后的小数点
         """
         # 先拼接需要保留的位数
         decimal_deal = f"%.{decimal_places}f"

diff --git a/ayugespidertools/items.py b/ayugespidertools/items.py
@@ -63,7 +63,7 @@ def asitem(self, assignment: bool = True) -> ScrapyItem:
                 assignment: 是否将 AyuItem 中的值赋值给 ScrapyItem，默认为 True
 
             Returns:
-                new_class: 转换 ScrapyItem 后的实例
+                1). 转换 ScrapyItem 后的实例
             """
             item_temp = ScrapyItem()
             for k, v in self.asdict().items():
@@ -177,11 +177,8 @@ def fields(self):
         self.__fields.discard("_AyuItem__fields")
         return self.__fields
 
-    def add_field(self, key: str, value: Any) -> None:
-        ...
+    def add_field(self, key: str, value: Any) -> None: ...
 
-    def asdict(self) -> Dict[str, Any]:
-        ...
+    def asdict(self) -> Dict[str, Any]: ...
 
-    def asitem(self, assignment: bool = True) -> ScrapyItem:
-        ...
+    def asitem(self, assignment: bool = True) -> ScrapyItem: ...
diff --git a/ayugespidertools/scraper/middlewares/headers/ua.py b/ayugespidertools/scraper/middlewares/headers/ua.py
@@ -40,7 +40,9 @@ def spider_opened(self, spider):
         ]
         self.explorer_types = [x["explorer"] for x in ua_arr]
         self.explorer_weights = [x["weight"] for x in ua_arr]
-        spider.slog.info(f"随机请求头中间件 RandomRequestUaMiddleware 已开启，生效脚本为: {spider.name}")
+        spider.slog.info(
+            f"随机请求头中间件 RandomRequestUaMiddleware 已开启，生效脚本为: {spider.name}"
+        )
 
     def process_request(self, request, spider):
         # 根据权重来获取随机请求头 ua 信息

diff --git a/ayugespidertools/scraper/middlewares/proxy/dynamic.py b/ayugespidertools/scraper/middlewares/proxy/dynamic.py
@@ -24,13 +24,13 @@ def process_request(self, request, spider):
         # TODO: 根据权重来随机获取一个账号 DYNAMIC_PROXY_CONFIG
         # account = ReuseOperation.random_weight(self.account_arr)
         if request.url.startswith("https://"):
-            request.meta[
-                "proxy"
-            ] = f"https://{self.username}:{self.password}@{self.proxy_url}/"
+            request.meta["proxy"] = (
+                f"https://{self.username}:{self.password}@{self.proxy_url}/"
+            )
         elif request.url.startswith("http://"):
-            request.meta[
-                "proxy"
-            ] = f"http://{self.username}:{self.password}@{self.proxy_url}/"
+            request.meta["proxy"] = (
+                f"http://{self.username}:{self.password}@{self.proxy_url}/"
+            )
         else:
             spider.slog.info(
                 f"request url: {request.url} error when use proxy middlewares!"
@@ -61,7 +61,9 @@ def __init__(self, settings):
             dict_conf=dynamic_proxy_conf,
             key_list=["proxy", "username", "password"],
         )
-        assert is_match, f"没有配置动态隧道代理，配置示例为：{Param.dynamic_proxy_conf_example}"
+        assert (
+            is_match
+        ), f"没有配置动态隧道代理，配置示例为：{Param.dynamic_proxy_conf_example}"
 
         self.proxy_url = dynamic_proxy_conf["proxy"]
         self.username = dynamic_proxy_conf["username"]

diff --git a/ayugespidertools/scraper/pipelines/es/fantasy.py b/ayugespidertools/scraper/pipelines/es/fantasy.py
@@ -3,5 +3,4 @@
 __all__ = ["AyuFtyESPipeline"]
 
 
-class AyuFtyESPipeline(AyuESPipeline):
-    ...
+class AyuFtyESPipeline(AyuESPipeline): ...
diff --git a/ayugespidertools/scraper/pipelines/mysql/fantasy.py b/ayugespidertools/scraper/pipelines/mysql/fantasy.py
@@ -3,5 +3,4 @@
 __all__ = ["AyuFtyMysqlPipeline"]
 
 
-class AyuFtyMysqlPipeline(AyuMysqlPipeline):
-    ...
+class AyuFtyMysqlPipeline(AyuMysqlPipeline): ...
diff --git a/ayugespidertools/scraper/pipelines/oracle/fantasy.py b/ayugespidertools/scraper/pipelines/oracle/fantasy.py
@@ -3,5 +3,4 @@
 __all__ = ["AyuFtyOraclePipeline"]
 
 
-class AyuFtyOraclePipeline(AyuOraclePipeline):
-    ...
+class AyuFtyOraclePipeline(AyuOraclePipeline): ...
diff --git a/ayugespidertools/scraper/pipelines/postgres/fantasy.py b/ayugespidertools/scraper/pipelines/postgres/fantasy.py
@@ -3,5 +3,4 @@
 __all__ = ["AyuFtyPostgresPipeline"]
 
 
-class AyuFtyPostgresPipeline(AyuPostgresPipeline):
-    ...
+class AyuFtyPostgresPipeline(AyuPostgresPipeline): ...