diff --git a/docs/changelog.rst b/docs/changelog.rst index 349c426..909ea28 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,20 @@ Change log ========== +0.7.1 (2017-?-?) +---------------- + +New features +~~~~~~~~~~~~ + + +Update +~~~~~~ + +- ``xpaw.utils.run`` 模块中 ``run_crawler`` 函数移动至 ``xpaw.run`` 模块 +- 合并utils为一个模块 + + 0.7.0 (2017-10-24) ------------------ @@ -10,7 +24,7 @@ New features ~~~~~~~~~~~~ - 使用继承Dupefilter的去重过滤器来实现去重功能,系统配置新增 ``dupefilter_cls`` 项,用于替换默认的去重过滤器 -- ``xpaw.utils`` 模块中新增 ``run_crawler`` 函数,便于在python代码中控制开启爬虫 +- ``xpaw.utils.run`` 模块中新增 ``run_crawler`` 函数,便于在python代码中控制开启爬虫 Update ~~~~~~ diff --git a/tests/test_downloader.py b/tests/test_downloader.py index b7554cc..6290bfb 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -3,7 +3,7 @@ import json import random -from xpaw import HttpRequest +from xpaw.http import HttpRequest from xpaw.downloader import Downloader, DownloaderMiddlewareManager from xpaw.downloadermws import CookieJarMiddleware diff --git a/xpaw/config/__init__.py b/xpaw/config.py similarity index 99% rename from xpaw/config/__init__.py rename to xpaw/config.py index 408d7f7..2705623 100644 --- a/xpaw/config/__init__.py +++ b/xpaw/config.py @@ -4,7 +4,7 @@ from collections import MutableMapping import types -from . import defaultconfig +from xpaw import defaultconfig CONFIG_PRIORITIES = { "default": 0, diff --git a/xpaw/config/defaultconfig.py b/xpaw/defaultconfig.py similarity index 100% rename from xpaw/config/defaultconfig.py rename to xpaw/defaultconfig.py diff --git a/xpaw/run.py b/xpaw/run.py new file mode 100644 index 0000000..7e39d1f --- /dev/null +++ b/xpaw/run.py @@ -0,0 +1,17 @@ +# coding=utf-8 + +from os.path import isfile, join, abspath + +from xpaw.config import Config +from xpaw.cluster import LocalCluster +from xpaw.utils import configure_logging + + +def run_crawler(project_dir, **kwargs): + if not isfile(join(project_dir, "setup.cfg")): + raise FileNotFoundError("Cannot find 'setup.cfg' in {}".format(abspath(project_dir))) + + config = Config(kwargs, priority="project") + configure_logging("xpaw", config) + cluster = LocalCluster(project_dir, config) + cluster.start() diff --git a/xpaw/utils.py b/xpaw/utils.py index 6f9c046..4fb19e1 100644 --- a/xpaw/utils.py +++ b/xpaw/utils.py @@ -7,12 +7,8 @@ import logging from importlib import import_module from pkgutil import iter_modules -from os.path import isfile, join, abspath import string -from xpaw.config import Config -from xpaw.cluster import LocalCluster - def load_object(path): dot = path.rindex(".") @@ -107,13 +103,3 @@ def render_templatefile(path, **kwargs): def string_camelcase(s): return _camelcase_invalid_chars.sub('', s.title()) - - -def run_crawler(project_dir, **kwargs): - if not isfile(join(project_dir, "setup.cfg")): - raise FileNotFoundError("Cannot find 'setup.cfg' in {}".format(abspath(project_dir))) - - config = Config(kwargs, priority="project") - configure_logging("xpaw", config) - cluster = LocalCluster(project_dir, config) - cluster.start()