From 91bc736b0a8dcabd3c85acbf66ca7c12191bcdf6 Mon Sep 17 00:00:00 2001 From: kem wan Date: Thu, 23 May 2024 12:00:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20by=20csv=20=E5=87=BA?= =?UTF-8?q?=E7=8E=B0=E9=95=BF=E8=B7=AF=E5=BE=84=E6=97=B6=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bricks/db/sqlite.py | 2 +- bricks/utils/csv_.py | 5 +++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bricks/db/sqlite.py b/bricks/db/sqlite.py index 6c887c3..0cb9b70 100644 --- a/bricks/db/sqlite.py +++ b/bricks/db/sqlite.py @@ -148,7 +148,7 @@ def load_csv( conn.execute(f'DROP TABLE IF EXISTS {table};') if structure: conn.create_table(table, structure=structure) - cmd = f'sqlite3 {database}.db ".mode csv" ".import {path} {table}"' + cmd = f'sqlite3 {database}.db ".mode csv" ".import {path!r} {table!r}"' options = {} if not debug: diff --git a/bricks/utils/csv_.py b/bricks/utils/csv_.py index a94d566..65075e8 100644 --- a/bricks/utils/csv_.py +++ b/bricks/utils/csv_.py @@ -8,6 +8,7 @@ import atexit import csv import functools +import hashlib import os.path import re import threading @@ -80,8 +81,8 @@ def iter_data(self, sql: str, batch_size: int = 10000): if not os.path.exists(path): raise FileNotFoundError(path) - table = NAME_PATTERN.sub("_", path[:-4]) - database = NAME_PATTERN.sub("_", self.path) + table = NAME_PATTERN.sub("_", os.path.basename(path)) + database = hashlib.md5(os.path.dirname(path).encode()).hexdigest() if self.structure: with open(path, encoding=self.encoding) as f: header = csv.DictReader(f, **self.options).fieldnames diff --git a/setup.py b/setup.py index 7a272d1..3798a08 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ def read_requirements(filename): url='https://github.com/KKKKKKKEM/bricks.git', long_description_content_type="text/markdown", long_description=open('README.md', encoding='utf-8').read(), - version="0.0.22.beta4", # 包的版本 + version="0.0.22.beta5", # 包的版本 description="quickly build your crawler", # 包简介,显示在PyPI上 author="Kem", # 作者相关信息 author_email='531144129@qq.com',