Fix deprecated CrawlerSettings and increase backwards compatibility with .defaults attribute

Author: dangra

scrapy/settings/__init__.py

@@ -126,14 +126,28 @@ def __setitem__(this, key, value):
             self._overrides = o = _DictProxy()
         return o
 
+    @property
+    def defaults(self):
+        warnings.warn("`Settings.defaults` attribute is deprecated and won't "
+                      "be supported in Scrapy 0.26, use "
+                      "`Settings.set(name, value, priority='default')` instead",
+                      category=ScrapyDeprecationWarning, stacklevel=2)
+        try:
+            o = self._defaults
+        except AttributeError:
+            class _DictProxy(dict):
+                def __setitem__(this, key, value):
+                    super(_DictProxy, this).__setitem__(key, value)
+                    self.set(key, value, priority='default')
+            self._defaults = o = _DictProxy()
+        return o
+
 
 class CrawlerSettings(Settings):
 
     def __init__(self, settings_module=None, **kw):
         Settings.__init__(self, **kw)
         self.settings_module = settings_module
-        self.overrides = {}
-        self.defaults = {}
 
     def __getitem__(self, opt_name):
         if opt_name in self.overrides:

scrapy/tests/test_settings/__init__.py

@@ -6,7 +6,7 @@
 except ImportError:
     import mock
 
-from scrapy.settings import Settings, SettingsAttribute
+from scrapy.settings import Settings, SettingsAttribute, CrawlerSettings
 from . import default_settings
 
 
@@ -190,7 +190,7 @@ def test_get(self):
         self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
         self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
 
-    def test_deprecated_attribute(self):
+    def test_deprecated_attribute_overrides(self):
         self.settings.set('BAR', 'fuz', priority='cmdline')
         with warnings.catch_warnings(record=True) as w:
             self.settings.overrides['BAR'] = 'foo'
@@ -199,6 +199,43 @@ def test_deprecated_attribute(self):
             self.assertEqual(self.settings.overrides.get('BAR'), 'foo')
             self.assertIn('BAR', self.settings.overrides)
 
+    def test_deprecated_attribute_defaults(self):
+        self.settings.set('BAR', 'fuz', priority='default')
+        with warnings.catch_warnings(record=True) as w:
+            self.settings.defaults['BAR'] = 'foo'
+            self.assertIn("Settings.defaults", str(w[0].message))
+            self.assertEqual(self.settings.get('BAR'), 'foo')
+            self.assertEqual(self.settings.defaults.get('BAR'), 'foo')
+            self.assertIn('BAR', self.settings.defaults)
+
+
+class CrawlerSettingsTest(unittest.TestCase):
+
+    def test_deprecated_crawlersettings(self):
+        def _get_settings(settings_dict=None):
+            settings_module = type('SettingsModuleMock', (object,), settings_dict or {})
+            return CrawlerSettings(settings_module)
+
+        with warnings.catch_warnings(record=True) as w:
+            settings = _get_settings()
+            self.assertIn("CrawlerSettings is deprecated", str(w[0].message))
+
+            # test_global_defaults
+            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180)
+
+            # test_defaults
+            settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
+            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99)
+
+            # test_settings_module
+            settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'})
+            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3)
+
+            # test_overrides
+            settings = _get_settings({'DOWNLOAD_TIMEOUT': '3'})
+            settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
+            self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15)
+
 
 if __name__ == "__main__":
     unittest.main()