Merge pull request #141 from Lukas0907/release

Release

feeds.cfg.dist

@@ -24,9 +24,9 @@ useragent = feeds (+https://github.com/nblock/feeds)
 # remove_images = 1
 
 ## Enable caching of responses
-# cache_enabled = 0
+# cache_enabled = 1
 ## Path to the cache.
-# cache_dir = .cache
+# cache_dir = ~/.cache/feeds
 ## Expire (remove) entries from cache after 14 days
 # cache_expires = 14
 

feeds/default_settings.py

@@ -1,7 +1,12 @@
 import logging
+import os
 
-# Feeds configuration populated by an optional feeds configuration file.
-FEEDS_CONFIG = {}
+from xdg.BaseDirectory import save_cache_path, xdg_config_home
+
+# Default settings for Feeds specific configurations.
+FEEDS_CONFIG_OUTPUT_PATH = "output"
+FEEDS_CONFIG_FILE = os.path.join(xdg_config_home, "feeds.cfg")
+FEEDS_CONFIG_CACHE_EXPIRES = 14
 
 # Low level settings intended for scrapy.
 # Please use feeds.cfg to configure feeds.
@@ -34,10 +39,10 @@
     "scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": None,
 }
 
-HTTPCACHE_ENABLED = False
+HTTPCACHE_ENABLED = True
 HTTPCACHE_STORAGE = "feeds.extensions.FeedsCacheStorage"
 HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
-HTTPCACHE_DIR = "cache"
+HTTPCACHE_DIR = save_cache_path("feeds")
 # We cache everything and delete cache entries (and every parent request) during
 # cleanup.
 HTTPCACHE_IGNORE_HTTP_CODES = []

feeds/pipelines.py

@@ -60,7 +60,9 @@ def __init__(self, output_path, output_url):
 
     @classmethod
     def from_crawler(cls, crawler):
-        output_path = crawler.settings.get("FEEDS_CONFIG_OUTPUT_PATH", "output")
+        output_path = crawler.settings.get("FEEDS_CONFIG_OUTPUT_PATH")
+        if not output_path:
+            raise ValueError("output_path not set!")
         output_url = crawler.settings.get("FEEDS_CONFIG_OUTPUT_URL")
         pipeline = cls(output_path=output_path, output_url=output_url)
         crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)

feeds/settings.py

@@ -8,11 +8,16 @@
 _SETTINGS = None
 
 
-def load_feeds_settings(file_=None):
+def load_feeds_settings(file_):
     settings = get_project_settings()
     set_feeds_settings(settings)
     if not file_:
-        return settings
+        config_file_path = settings.get("FEEDS_CONFIG_FILE")
+        try:
+            file_ = open(config_file_path, "r")
+        except IOError:
+            logger.info("Could not load config file from {}!".format(config_file_path))
+            return settings
 
     logger.debug("Parsing configuration file {} ...".format(file_.name))
     # Parse configuration file and store result under FEEDS_CONFIG of scrapy's
@@ -39,9 +44,11 @@ def load_feeds_settings(file_=None):
         "HTTPCACHE_DIR": config.get("feeds", "cache_dir", fallback=None),
     }
     for key, value in feeds_cfgfile_mapping.items():
-        if value:
+        if value is not None:
             settings.set(key, value)
 
+    file_.close()
+
     return settings
 
 

setup.py

@@ -17,6 +17,7 @@
         "feedparser",
         "lxml>=3.5.0",
         "python-dateutil>=2.7.3",
+        "pyxdg>=0.26",
         "readability-lxml>=0.7",
     ],
     extras_require={