bug 1401246: Update sample database

Updates for a sample DB regen just before the Django 1.11 update:

* Try 7 times for the very slow Midas/Security_Preferences history
* Drop the interactive editor content experiment pages
* Add resources needed by headless testing for CDN headers
* Adjust a headless CDN test to use a real revision

etc/sample_db.json

@@ -1,7 +1,6 @@
 {
   "sources": [
     ["links", "/", {"translations": true, "revisions": 3}],
-    ["links", "/en-US/docs/Experiment:InteractiveEditor", {}],
     ["document", "/en-US/docs/Mozilla/Firefox", {"revisions": 15}],
     ["document", "/en-US/docs/Web/Apps", {"translations": true, "revisions": 3}],
     ["document", "/en-US/docs/MDN_at_ten/History_of_MDN", {}],
@@ -14,6 +13,13 @@
     ["document", "/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach", {}],
     ["document", "/en-US/docs/Web/Guide/CSS/Media_queries", {}],
     ["document", "/en-US/docs/Learn/Getting_started_with_the_web/JavaScript_basics", {}],
+    ["document", "/en-US/docs/Learn/CSS/Styling_text/Fundamentals", {}],
+    ["document", "/es/docs/Mozilla/Firefox", {"revisions": 3}],
+    ["document", "/fr/docs/Mozilla/Firefox", {"revisions": 3}],
+    ["document", "/de/docs/Mozilla/Firefox", {"revisions": 3}],
+    ["revision", "/en-US/docs/Web/HTML$revision/1299417", {}],
+    ["revision", "/en-US/docs/Web/HTML$revision/1293895", {}],
+    ["revision", "/en-US/docs/Mozilla/Firefox$revision/1284393", {}],
     ["user", "Sheppy", {"social": true, "email": "sheppy@mozilla.com"}],
     ["user", "jwhitlock", {"social": true, "email": "jwhitlock@mozilla.com"}]
   ],

kuma/scrape/scraper.py

@@ -19,7 +19,7 @@
 class Requester(object):
     """Request pages from a running MDN instance."""
 
-    MAX_ATTEMPTS = 4
+    MAX_ATTEMPTS = 7
     TIMEOUT = 1.0
 
     def __init__(self, host, ssl):

kuma/scrape/tests/test_scraper.py

@@ -99,24 +99,20 @@ def test_connectionerror_success():
 @mock.patch('kuma.scrape.scraper.time.sleep')
 def test_timeout_failure(mock_sleep):
     """Request fail after too many Timeouts."""
-    assert Requester.MAX_ATTEMPTS == 4
+    attempts = 7
+    assert Requester.MAX_ATTEMPTS == attempts
     requester = Requester('example.com', True)
     mock_session = mock.Mock(spec_set=['get'])
-    mock_session.get.side_effect = [requests.exceptions.Timeout] * 4
+    mock_session.get.side_effect = [requests.exceptions.Timeout] * attempts
     requester._session = mock_session
 
     with pytest.raises(requests.exceptions.Timeout):
         requester.request('/path', raise_for_status=False)
     full_path = 'https://example.com/path'
-    expected_calls = [
-        mock.call(full_path, timeout=1.0),
-        mock.call(full_path, timeout=2.0),
-        mock.call(full_path, timeout=4.0),
-        mock.call(full_path, timeout=8.0),
-    ]
+    times = [2.0 ** attempt for attempt in range(attempts)]  # 1, 2, 4, 8...
+    expected_calls = [mock.call(full_path, timeout=time) for time in times]
     assert mock_session.get.call_args_list == expected_calls
-    expected_sleep_calls = [mock.call(1.0), mock.call(2.0), mock.call(4.0),
-                            mock.call(8.0)]
+    expected_sleep_calls = [mock.call(time) for time in times]
     assert mock_sleep.call_args_list == expected_sleep_calls
 
 

tests/headless/test_cdn.py

@@ -155,7 +155,7 @@ def test_maintenance_mode(base_url, is_behind_cdn, is_maintenance_mode):
              '/en-US/Firefox$delete',
              '/en-US/Firefox$translate',
              '/en-US/Firefox$quick-review',
-             '/en-US/Firefox$revert/1358677'])
+             '/en-US/Firefox$revert/1284393'])
 def test_not_cached_login_required(base_url, is_behind_cdn, slug):
     """Ensure that these endpoints that require login are not cached."""
     url = base_url + slug
@@ -415,10 +415,6 @@ def test_locale_selection_cached(base_url, is_behind_cdn, is_local_url, slug,
     assert expected, "expected must be set to the expected locale prefix."
     assert accept, "accept must be set to the Accept-Langauge header value."
 
-    if (is_local_url and slug.startswith('/Firefox') and
-            expected in ('fr', 'es')):
-        pytest.xfail("/%s/Firefox is not in sample database." % expected)
-
     request_kwargs = {
         'headers': {
             'X-Requested-With': 'XMLHttpRequest',
@@ -459,7 +455,7 @@ def test_locale_selection_cached(base_url, is_behind_cdn, is_local_url, slug,
              '/Firefox$quick-review',
              '/Firefox$subscribe',
              '/Firefox$subscribe_to_tree',
-             '/Firefox$revert/1358677'])
+             '/Firefox$revert/1284393'])
 def test_locale_selection_not_cached(base_url, is_behind_cdn, is_local_url,
                                      slug, expected, accept, cookie, param):
     """
@@ -473,10 +469,6 @@ def test_locale_selection_not_cached(base_url, is_behind_cdn, is_local_url,
     assert expected, "expected must be set to the expected locale prefix."
     assert accept, "accept must be set to the Accept-Langauge header value."
 
-    if (is_local_url and slug.startswith('/Firefox') and
-            expected in ('fr', 'es')):
-        pytest.xfail("/%s/Firefox is not in sample database." % expected)
-
     request_kwargs = {
         'headers': {
             'X-Requested-With': 'XMLHttpRequest',