[MRG+1] Make data_path work when outside project (used by HttpCacheMiddleware and Deltafetch plugin)

scrapy/utils/project.py

@@ -12,6 +12,7 @@
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
 
+
 def inside_project():
     scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
@@ -23,6 +24,7 @@ def inside_project():
             return True
     return bool(closest_scrapy_cfg())
 
+
 def project_data_dir(project='default'):
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
@@ -39,16 +41,22 @@ def project_data_dir(project='default'):
         os.makedirs(d)
     return d
 
+
 def data_path(path, createdir=False):
-    """If path is relative, return the given path inside the project data dir,
-    otherwise return the path unmodified
+    """
+    Return the given path joined with the .scrapy data directory.
+    If given an absolute path, return it unmodified.
     """
     if not isabs(path):
-        path = join(project_data_dir(), path)
+        if inside_project():
+            path = join(project_data_dir(), path)
+        else:
+            path = join('.scrapy', path)
     if createdir and not exists(path):
         os.makedirs(path)
     return path
 
+
 def get_project_settings():
     if ENVVAR not in os.environ:
         project = os.environ.get('SCRAPY_PROJECT', 'default')

tests/test_utils_project.py

@@ -0,0 +1,34 @@
+import unittest
+import os
+import tempfile
+import shutil
+import contextlib
+from scrapy.utils.project import data_path
+
+
+@contextlib.contextmanager
+def inside_a_project():
+    prev_dir = os.getcwd()
+    project_dir = tempfile.mkdtemp()
+
+    try:
+        os.chdir(project_dir)
+        with open('scrapy.cfg', 'w') as f:
+            # create an empty scrapy.cfg
+            f.close()
+
+        yield project_dir
+    finally:
+        os.chdir(prev_dir)
+        shutil.rmtree(project_dir)
+
+
+class ProjectUtilsTest(unittest.TestCase):
+    def test_data_path_outside_project(self):
+        self.assertEquals('.scrapy/somepath', data_path('somepath'))
+
+    def test_data_path_inside_project(self):
+        with inside_a_project() as proj_path:
+            expected = os.path.join(proj_path, '.scrapy', 'somepath')
+            self.assertEquals(expected, data_path('somepath'))
+            self.assertEquals('/absolute/path', data_path('/absolute/path'))