Auto merge of #12441 - aravind-pg:referrer-pol-header, r=jdm

Implement referrer policy delivery by header

Adds a new `Option<ReferrerPolicy>` field to Document and sets it appropriately in `ScriptThread::load` if a Referrer-Policy header is present.

r? @jdm

<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #11860
- [X] There are tests for these changes

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12441)
<!-- Reviewable:end -->

components/msg/constellation_msg.rs

@@ -332,7 +332,7 @@ pub enum FrameType {
 #[derive(Clone, Copy, Debug, Deserialize, HeapSizeOf, Serialize)]
 pub enum ReferrerPolicy {
     NoReferrer,
-    NoRefWhenDowngrade,
+    NoReferrerWhenDowngrade,
     Origin,
     SameOrigin,
     OriginWhenCrossOrigin,

components/net/fetch/methods.rs

@@ -155,7 +155,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
 
     // Step 7
     if request.referrer_policy.get().is_none() {
-        request.referrer_policy.set(Some(ReferrerPolicy::NoRefWhenDowngrade));
+        request.referrer_policy.set(Some(ReferrerPolicy::NoReferrerWhenDowngrade));
     }
 
     // Step 8

components/net/http_loader.rs

@@ -425,7 +425,7 @@ fn set_default_accept_language(headers: &mut Headers) {
 }
 
 /// https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-state-no-referrer-when-downgrade
-fn no_ref_when_downgrade_header(referrer_url: Url, url: Url) -> Option<Url> {
+fn no_referrer_when_downgrade_header(referrer_url: Url, url: Url) -> Option<Url> {
     if referrer_url.scheme() == "https" && url.scheme() != "https" {
         return None;
     }
@@ -462,7 +462,8 @@ pub fn determine_request_referrer(headers: &mut Headers,
             Some(ReferrerPolicy::SameOrigin) => if cross_origin { None } else { strip_url(ref_url, false) },
             Some(ReferrerPolicy::UnsafeUrl) => strip_url(ref_url, false),
             Some(ReferrerPolicy::OriginWhenCrossOrigin) => strip_url(ref_url, cross_origin),
-            Some(ReferrerPolicy::NoRefWhenDowngrade) | None => no_ref_when_downgrade_header(ref_url, url),
+            Some(ReferrerPolicy::NoReferrerWhenDowngrade) | None =>
+                no_referrer_when_downgrade_header(ref_url, url),
         };
     }
     return None;

components/script/dom/document.rs

@@ -1633,7 +1633,8 @@ impl Document {
                          last_modified: Option<String>,
                          source: DocumentSource,
                          doc_loader: DocumentLoader,
-                         referrer: Option<String>)
+                         referrer: Option<String>,
+                         referrer_policy: Option<ReferrerPolicy>)
                          -> Document {
         let url = url.unwrap_or_else(|| Url::parse("about:blank").unwrap());
 
@@ -1652,6 +1653,17 @@ impl Document {
             Origin::opaque_identifier()
         };
 
+        // TODO: we currently default to Some(NoReferrer) instead of None (i.e. unset)
+        // for an important reason. Many of the methods by which a referrer policy is communicated
+        // are currently unimplemented, and so in such cases we may be ignoring the desired policy.
+        // If the default were left unset, then in Step 7 of the Fetch algorithm we adopt
+        // no-referrer-when-downgrade. However, since we are potentially ignoring a stricter
+        // referrer policy, this might be passing too much info. Hence, we default to the
+        // strictest policy, which is no-referrer.
+        // Once other delivery methods are implemented, make the unset case really
+        // unset (i.e. None).
+        let referrer_policy = referrer_policy.or(Some(ReferrerPolicy::NoReferrer));
+
         Document {
             node: Node::new_document_node(),
             window: JS::from_ref(window),
@@ -1718,9 +1730,8 @@ impl Document {
             https_state: Cell::new(HttpsState::None),
             touchpad_pressure_phase: Cell::new(TouchpadPressurePhase::BeforeClick),
             origin: origin,
-            //TODO - setting this for now so no Referer header set
-            referrer_policy: Cell::new(Some(ReferrerPolicy::NoReferrer)),
             referrer: referrer,
+            referrer_policy: Cell::new(referrer_policy),
         }
     }
 
@@ -1738,6 +1749,7 @@ impl Document {
                          None,
                          DocumentSource::NotFromParser,
                          docloader,
+                         None,
                          None))
     }
 
@@ -1749,7 +1761,8 @@ impl Document {
                last_modified: Option<String>,
                source: DocumentSource,
                doc_loader: DocumentLoader,
-               referrer: Option<String>)
+               referrer: Option<String>,
+               referrer_policy: Option<ReferrerPolicy>)
                -> Root<Document> {
         let document = reflect_dom_object(box Document::new_inherited(window,
                                                                       browsing_context,
@@ -1759,7 +1772,8 @@ impl Document {
                                                                       last_modified,
                                                                       source,
                                                                       doc_loader,
-                                                                      referrer),
+                                                                      referrer,
+                                                                      referrer_policy),
                                           GlobalRef::Window(window),
                                           DocumentBinding::Wrap);
         {
@@ -1824,6 +1838,7 @@ impl Document {
                                         None,
                                         DocumentSource::NotFromParser,
                                         DocumentLoader::new(&self.loader()),
+                                        None,
                                         None);
             new_doc.appropriate_template_contents_owner_document.set(Some(&new_doc));
             new_doc
@@ -2848,7 +2863,7 @@ pub fn determine_policy_for_token(token: &str) -> Option<ReferrerPolicy> {
     let lower = token.to_lowercase();
     return match lower.as_ref() {
         "never" | "no-referrer" => Some(ReferrerPolicy::NoReferrer),
-        "default" | "no-referrer-when-downgrade" => Some(ReferrerPolicy::NoRefWhenDowngrade),
+        "default" | "no-referrer-when-downgrade" => Some(ReferrerPolicy::NoReferrerWhenDowngrade),
         "origin" => Some(ReferrerPolicy::Origin),
         "same-origin" => Some(ReferrerPolicy::SameOrigin),
         "origin-when-cross-origin" => Some(ReferrerPolicy::OriginWhenCrossOrigin),

components/script/dom/domimplementation.rs

@@ -130,6 +130,7 @@ impl DOMImplementationMethods for DOMImplementation {
                                 None,
                                 DocumentSource::NotFromParser,
                                 loader,
+                                None,
                                 None);
 
         {

components/script/dom/domparser.rs

@@ -69,6 +69,7 @@ impl DOMParserMethods for DOMParser {
                                              None,
                                              DocumentSource::FromParser,
                                              loader,
+                                             None,
                                              None);
                 parse_html(document.r(), s, url, ParseContext::Owner(None));
                 document.set_ready_state(DocumentReadyState::Complete);
@@ -84,6 +85,7 @@ impl DOMParserMethods for DOMParser {
                                              None,
                                              DocumentSource::NotFromParser,
                                              loader,
+                                             None,
                                              None);
                 parse_xml(document.r(), s, url, xml::ParseContext::Owner(None));
                 Ok(document)

components/script/dom/node.rs

@@ -1721,7 +1721,8 @@ impl Node {
                 let document = Document::new(window, None,
                                              Some((*document.url()).clone()),
                                              is_html_doc, None,
-                                             None, DocumentSource::NotFromParser, loader, None);
+                                             None, DocumentSource::NotFromParser, loader,
+                                             None, None);
                 Root::upcast::<Node>(document)
             },
             NodeTypeId::Element(..) => {

components/script/dom/xmldocument.rs

@@ -42,6 +42,7 @@ impl XMLDocument {
                                               last_modified,
                                               source,
                                               doc_loader,
+                                              None,
                                               None),
         }
     }

components/script/dom/xmlhttprequest.rs

@@ -1242,6 +1242,7 @@ impl XMLHttpRequest {
                       None,
                       DocumentSource::FromParser,
                       docloader,
+                      None,
                       None)
     }
 

components/script/parse/html.rs

@@ -280,7 +280,7 @@ pub fn parse_html_fragment(context_node: &Node,
                                  None, None,
                                  DocumentSource::FromParser,
                                  loader,
-                                 None);
+                                 None, None);
 
     // Step 2.
     document.set_quirks_mode(context_document.quirks_mode());

components/script/script_thread.rs

@@ -51,8 +51,8 @@ use dom::worker::TrustedWorkerAddress;
 use euclid::Rect;
 use euclid::point::Point2D;
 use gfx_traits::LayerId;
-use hyper::header::{ContentType, HttpDate};
-use hyper::header::{Headers, LastModified};
+use hyper::header::{ContentType, Headers, HttpDate, LastModified};
+use hyper::header::{ReferrerPolicy as ReferrerPolicyHeader};
 use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
 use ipc_channel::ipc::{self, IpcSender};
@@ -65,7 +65,7 @@ use js::jsval::UndefinedValue;
 use js::rust::Runtime;
 use mem::heap_size_of_self_and_children;
 use msg::constellation_msg::{FrameType, LoadData, PanicMsg, PipelineId, PipelineNamespace};
-use msg::constellation_msg::{SubpageId, WindowSizeType};
+use msg::constellation_msg::{ReferrerPolicy, SubpageId, WindowSizeType};
 use net_traits::LoadData as NetLoadData;
 use net_traits::bluetooth_thread::BluetoothMethodMsg;
 use net_traits::image_cache_thread::{ImageCacheChan, ImageCacheResult, ImageCacheThread};
@@ -1716,6 +1716,25 @@ impl ScriptThread {
             None => None,
         };
 
+        let referrer_policy = if let Some(headers) = metadata.headers {
+            headers.get::<ReferrerPolicyHeader>().map(|h| match *h {
+                ReferrerPolicyHeader::NoReferrer =>
+                    ReferrerPolicy::NoReferrer,
+                ReferrerPolicyHeader::NoReferrerWhenDowngrade =>
+                    ReferrerPolicy::NoReferrerWhenDowngrade,
+                ReferrerPolicyHeader::SameOrigin =>
+                    ReferrerPolicy::SameOrigin,
+                ReferrerPolicyHeader::Origin =>
+                    ReferrerPolicy::Origin,
+                ReferrerPolicyHeader::OriginWhenCrossOrigin =>
+                    ReferrerPolicy::OriginWhenCrossOrigin,
+                ReferrerPolicyHeader::UnsafeUrl =>
+                    ReferrerPolicy::UnsafeUrl,
+            })
+        } else {
+            None
+        };
+
         let document = Document::new(window.r(),
                                      Some(&browsing_context),
                                      Some(final_url.clone()),
@@ -1724,7 +1743,8 @@ impl ScriptThread {
                                      last_modified,
                                      DocumentSource::FromParser,
                                      loader,
-                                     referrer);
+                                     referrer,
+                                     referrer_policy);
         if using_new_context {
             browsing_context.init(&document);
         } else {