update

docs/changelog.rst

@@ -3,14 +3,28 @@
 Change log
 ==========
 
+0.7.1 (2017-?-?)
+----------------
+
+New features
+~~~~~~~~~~~~
+
+
+Update
+~~~~~~
+
+- ``xpaw.utils.run`` 模块中 ``run_crawler`` 函数移动至 ``xpaw.run`` 模块
+- 合并utils为一个模块
+
+
 0.7.0 (2017-10-24)
 ------------------
 
 New features
 ~~~~~~~~~~~~
 
 - 使用继承Dupefilter的去重过滤器来实现去重功能，系统配置新增 ``dupefilter_cls`` 项，用于替换默认的去重过滤器
-- ``xpaw.utils`` 模块中新增 ``run_crawler`` 函数，便于在python代码中控制开启爬虫
+- ``xpaw.utils.run`` 模块中新增 ``run_crawler`` 函数，便于在python代码中控制开启爬虫
 
 Update
 ~~~~~~

tests/test_downloader.py

@@ -3,7 +3,7 @@
 import json
 import random
 
-from xpaw import HttpRequest
+from xpaw.http import HttpRequest
 from xpaw.downloader import Downloader, DownloaderMiddlewareManager
 from xpaw.downloadermws import CookieJarMiddleware
 

xpaw/config/__init__.py → xpaw/config.py

@@ -4,7 +4,7 @@
 from collections import MutableMapping
 import types
 
-from . import defaultconfig
+from xpaw import defaultconfig
 
 CONFIG_PRIORITIES = {
     "default": 0,

xpaw/run.py

@@ -0,0 +1,17 @@
+# coding=utf-8
+
+from os.path import isfile, join, abspath
+
+from xpaw.config import Config
+from xpaw.cluster import LocalCluster
+from xpaw.utils import configure_logging
+
+
+def run_crawler(project_dir, **kwargs):
+    if not isfile(join(project_dir, "setup.cfg")):
+        raise FileNotFoundError("Cannot find 'setup.cfg' in {}".format(abspath(project_dir)))
+
+    config = Config(kwargs, priority="project")
+    configure_logging("xpaw", config)
+    cluster = LocalCluster(project_dir, config)
+    cluster.start()

xpaw/utils.py

@@ -7,12 +7,8 @@
 import logging
 from importlib import import_module
 from pkgutil import iter_modules
-from os.path import isfile, join, abspath
 import string
 
-from xpaw.config import Config
-from xpaw.cluster import LocalCluster
-
 
 def load_object(path):
     dot = path.rindex(".")
@@ -107,13 +103,3 @@ def render_templatefile(path, **kwargs):
 
 def string_camelcase(s):
     return _camelcase_invalid_chars.sub('', s.title())
-
-
-def run_crawler(project_dir, **kwargs):
-    if not isfile(join(project_dir, "setup.cfg")):
-        raise FileNotFoundError("Cannot find 'setup.cfg' in {}".format(abspath(project_dir)))
-
-    config = Config(kwargs, priority="project")
-    configure_logging("xpaw", config)
-    cluster = LocalCluster(project_dir, config)
-    cluster.start()