submit_selected(): Referer fixes

Don't override a Referer field that a user passes in.
Note that since HTTP headers are case-insensitive, use requests'
CaseInsensitiveDict to handle this.

Furthermore, don't modify the caller's **kwargs, the caller should
assume we do not modify the submitted dict, so make a copy
first.

Do this in _merge_referer() helper function, as we anticipate needing
this in other functions when MechanicalSoup#362 is fixed.

Add a test for overriding Referers.

mechanicalsoup/stateful_browser.py

@@ -8,6 +8,8 @@
 from .form import Form
 from .utils import LinkNotFoundError
 
+from requests.structures import CaseInsensitiveDict
+
 
 class _BrowserState:
     def __init__(self, page=None, url=None, form=None, request=None):
@@ -222,6 +224,18 @@ def select_form(self, selector="form", nr=0):
 
         return self.form
 
+    def _merge_referer(self, **kwargs):
+        """Helper function to set the Referer header in kwargs passed to
+        requests, if it has not already been overriden by the user."""
+
+        referer = self.url
+        headers = CaseInsensitiveDict(kwargs.get('headers', {}))
+        if referer is not None and 'Referer' not in headers:
+            kwargs = kwargs.copy()
+            headers['Referer'] = referer
+            kwargs['headers'] = headers
+        return kwargs
+
     def submit_selected(self, btnName=None, update_state=True,
                         *args, **kwargs):
         """Submit the form that was selected with :func:`select_form`.
@@ -237,13 +251,7 @@ def submit_selected(self, btnName=None, update_state=True,
         """
         self.form.choose_submit(btnName)
 
-        referer = self.url
-        if referer is not None:
-            if 'headers' in kwargs:
-                kwargs['headers']['Referer'] = referer
-            else:
-                kwargs['headers'] = {'Referer': referer}
-
+        kwargs = self._merge_referer(**kwargs)
         resp = self.submit(self.__state.form, url=self.__state.url,
                            *args, **kwargs)
         if update_state:

tests/test_stateful_browser.py

@@ -446,6 +446,20 @@ def test_referer_submit(httpbin):
     assert actual_ref == ref
 
 
+def test_referer_submit_override(httpbin):
+    browser = mechanicalsoup.StatefulBrowser()
+    ref = "https://example.com/my-referer"
+    ref_override = "https://example.com/override"
+    page = submit_form_headers.format(httpbin.url + "/headers")
+    browser.open_fake_page(page, url=ref)
+    browser.select_form()
+    response = browser.submit_selected(headers={"referer": ref_override})
+    headers = response.json()["headers"]
+    referer = headers["Referer"]
+    actual_ref = re.sub('/*$', '', referer)
+    assert actual_ref == ref_override
+
+
 def test_referer_submit_headers(httpbin):
     browser = mechanicalsoup.StatefulBrowser()
     ref = "https://example.com/my-referer"