Skip to content

Commit

Permalink
修复 by csv 出现长路径时报错的问题
Browse files Browse the repository at this point in the history
  • Loading branch information
kem wan committed May 23, 2024
1 parent 4cc3197 commit 91bc736
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 4 deletions.
2 changes: 1 addition & 1 deletion bricks/db/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def load_csv(
conn.execute(f'DROP TABLE IF EXISTS {table};')
if structure:
conn.create_table(table, structure=structure)
cmd = f'sqlite3 {database}.db ".mode csv" ".import {path} {table}"'
cmd = f'sqlite3 {database}.db ".mode csv" ".import {path!r} {table!r}"'

options = {}
if not debug:
Expand Down
5 changes: 3 additions & 2 deletions bricks/utils/csv_.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import atexit
import csv
import functools
import hashlib
import os.path
import re
import threading
Expand Down Expand Up @@ -80,8 +81,8 @@ def iter_data(self, sql: str, batch_size: int = 10000):
if not os.path.exists(path):
raise FileNotFoundError(path)

table = NAME_PATTERN.sub("_", path[:-4])
database = NAME_PATTERN.sub("_", self.path)
table = NAME_PATTERN.sub("_", os.path.basename(path))
database = hashlib.md5(os.path.dirname(path).encode()).hexdigest()
if self.structure:
with open(path, encoding=self.encoding) as f:
header = csv.DictReader(f, **self.options).fieldnames
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def read_requirements(filename):
url='https://github.com/KKKKKKKEM/bricks.git',
long_description_content_type="text/markdown",
long_description=open('README.md', encoding='utf-8').read(),
version="0.0.22.beta4", # 包的版本
version="0.0.22.beta5", # 包的版本
description="quickly build your crawler", # 包简介,显示在PyPI上
author="Kem", # 作者相关信息
author_email='531144129@qq.com',
Expand Down

0 comments on commit 91bc736

Please sign in to comment.