Bug 2000765 - Updated q-value generation in Accept-Language Header to match chrome.r=valentin,necko-reviewers,devtools-reviewers,android-reviewers,nchevobbe,nalexander

The algorithm now generates q-value 0.9 for second language in two language option. Now q-weight of each language decrements by 0.1 (minimum 0.1) down the language list.

Differential Revision: https://phabricator.services.mozilla.com/D273761

Author: Witty-31-06

devtools/client/netmonitor/test/browser_net_copy_headers.js

@@ -43,7 +43,7 @@ add_task(async function () {
     "Host: example.com",
     "User-Agent: " + navigator.userAgent + "",
     "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "Accept-Language: " + navigator.languages.join(",") + ";q=0.5",
+    "Accept-Language: " + navigator.languages.join(",") + ";q=0.9",
     "Accept-Encoding: gzip, deflate",
     "Connection: keep-alive",
     "Upgrade-Insecure-Requests: 1",

devtools/client/shared/test/xpcshell/test_curl.js

@@ -24,7 +24,7 @@ add_task(async function () {
           "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0",
       },
       { name: "Accept", value: "*/*" },
-      { name: "Accept-Language", value: "en-US,en;q=0.5" },
+      { name: "Accept-Language", value: "en-US,en;q=0.9" },
       { name: "Accept-Encoding", value: "gzip, deflate, br" },
       { name: "Origin", value: "https://example.com" },
       { name: "Connection", value: "keep-alive" },
@@ -54,7 +54,7 @@ add_task(async function () {
     "user-agent header present in curl command"
   );
   ok(
-    exactHeaderInParams(curlParams, "Accept-Language: en-US,en;q=0.5"),
+    exactHeaderInParams(curlParams, "Accept-Language: en-US,en;q=0.9"),
     "accept-language header present in curl output"
   );
   ok(

mobile/android/android-components/components/browser/engine-gecko/src/test/java/mozilla/components/browser/engine/gecko/fetch/GeckoViewFetchUnitTestCases.kt

@@ -117,7 +117,7 @@ class GeckoViewFetchUnitTestCases : FetchTestCases() {
         val requestHeaders = mapOf(
             "Accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Encoding" to "gzip, deflate",
-            "Accept-Language" to "en-US,en;q=0.5",
+            "Accept-Language" to "en-US,en;q=0.9",
             "Connection" to "keep-alive",
             "User-Agent" to "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0",
         )

mobile/android/android-components/components/concept/fetch/src/test/java/mozilla/components/concept/fetch/HeadersTest.kt

@@ -18,7 +18,7 @@ class HeadersTest {
         val headers = MutableHeaders(
             "Accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Encoding" to "gzip, deflate",
-            "Accept-Language" to "en-US,en;q=0.5",
+            "Accept-Language" to "en-US,en;q=0.9",
             "Connection" to "keep-alive",
             "Dnt" to "1",
             "User-Agent" to "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0",
@@ -35,7 +35,7 @@ class HeadersTest {
 
         assertEquals("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", headers[0].value)
         assertEquals("gzip, deflate", headers[1].value)
-        assertEquals("en-US,en;q=0.5", headers[2].value)
+        assertEquals("en-US,en;q=0.9", headers[2].value)
         assertEquals("keep-alive", headers[3].value)
         assertEquals("1", headers[4].value)
         assertEquals("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0", headers[5].value)
@@ -92,15 +92,15 @@ class HeadersTest {
         }
 
         headers[2] = Header("Dnt", "0")
-        headers[0] = Header("Accept-Language", "en-US,en;q=0.5")
+        headers[0] = Header("Accept-Language", "en-US,en;q=0.9")
 
         assertEquals(3, headers.size)
 
         assertEquals("Accept-Language", headers[0].name)
         assertEquals("Connection", headers[1].name)
         assertEquals("Dnt", headers[2].name)
 
-        assertEquals("en-US,en;q=0.5", headers[0].value)
+        assertEquals("en-US,en;q=0.9", headers[0].value)
         assertEquals("keep-alive", headers[1].value)
         assertEquals("0", headers[2].value)
     }
@@ -166,7 +166,7 @@ class HeadersTest {
         val headers = MutableHeaders(
             "Accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Encoding" to "gzip, deflate",
-            "Accept-Language" to "en-US,en;q=0.5",
+            "Accept-Language" to "en-US,en;q=0.9",
             "Connection" to "keep-alive",
             "Dnt" to "1",
             "User-Agent" to "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0",
@@ -188,7 +188,7 @@ class HeadersTest {
 
         assertEquals("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", headers[0].value)
         assertEquals("gzip, deflate", headers[1].value)
-        assertEquals("en-US,en;q=0.5", headers[2].value)
+        assertEquals("en-US,en;q=0.9", headers[2].value)
         assertEquals("keep-alive", headers[3].value)
         assertEquals("0", headers[4].value)
         assertEquals("Mozilla/6.0", headers[5].value)

mobile/android/android-components/components/concept/fetch/src/test/java/mozilla/components/concept/fetch/RequestTest.kt

@@ -38,7 +38,7 @@ class RequestTest {
             url = "https://www.mozilla.org",
             method = Request.Method.POST,
             headers = MutableHeaders(
-                "Accept-Language" to "en-US,en;q=0.5",
+                "Accept-Language" to "en-US,en;q=0.9",
                 "Connection" to "keep-alive",
                 "Dnt" to "1",
             ),
@@ -75,7 +75,7 @@ class RequestTest {
         assertEquals("Connection", headers[1].name)
         assertEquals("Dnt", headers[2].name)
 
-        assertEquals("en-US,en;q=0.5", headers[0].value)
+        assertEquals("en-US,en;q=0.9", headers[0].value)
         assertEquals("keep-alive", headers[1].value)
         assertEquals("1", headers[2].value)
     }

mobile/android/android-components/components/tooling/fetch-tests/src/main/java/mozilla/components/tooling/fetch/tests/FetchTestCases.kt

@@ -84,7 +84,7 @@ abstract class FetchTestCases {
                     headers = MutableHeaders()
                         .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
                         .set("Accept-Encoding", "gzip, deflate")
-                        .set("Accept-Language", "en-US,en;q=0.5")
+                        .set("Accept-Language", "en-US,en;q=0.9")
                         .set("Connection", "keep-alive")
                         .set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0"),
                 ),
@@ -113,7 +113,7 @@ abstract class FetchTestCases {
             )
 
             assertEquals(
-                "en-US,en;q=0.5",
+                "en-US,en;q=0.9",
                 request.headers.get("Accept-Language"),
             )
 

netwerk/base/rust-helper/src/lib.rs

@@ -67,14 +67,14 @@ fn open_read_fast_fail(path: &Path) -> io::Result<File> {
 /// Allocates an nsACString that contains a ISO 639 language list
 /// notated with HTTP "q" values for output with an HTTP Accept-Language
 /// header. Previous q values will be stripped because the order of
-/// the langs implies the q value. The q values are calculated by dividing
-/// 1.0 amongst the number of languages present.
+/// the langs implies the q value. q-values decrease by 0.1 for each subsequent language,
+/// with a minimum value of 0.1.
 ///
 /// Ex: passing: "en, ja"
-///     returns: "en,ja;q=0.5"
+///     returns: "en,ja;q=0.9"
 ///
 ///     passing: "en, ja, fr_CA"
-///     returns: "en,ja;q=0.7,fr_CA;q=0.3"
+///     returns: "en,ja;q=0.9,fr_CA;q=0.8"
 pub extern "C" fn rust_prepare_accept_languages(
     i_accept_languages: &nsACString,
     o_accept_languages: &mut nsACString,
@@ -90,8 +90,6 @@ pub extern "C" fn rust_prepare_accept_languages(
             .filter(|token| !token.is_empty())
     };
 
-    let n = make_tokens().count();
-
     for (count_n, i_token) in make_tokens().enumerate() {
         // delimiter if not first item
         if count_n != 0 {
@@ -106,23 +104,14 @@ pub extern "C" fn rust_prepare_accept_languages(
             canonicalize_language_tag(&mut o_token[token_pos..]);
         }
 
-        // Divide the quality-values evenly among the languages.
-        let q = 1.0 - count_n as f32 / n as f32;
-
-        let u: u32 = ((q + 0.005) * 100.0) as u32;
-        // Only display q-value if less than 1.00.
-        if u < 100 {
-            // With a small number of languages, one decimal place is
-            // enough to prevent duplicate q-values.
-            // Also, trailing zeroes do not add any information, so
-            // they can be removed.
-            if n < 10 || u % 10 == 0 {
-                let u = (u + 5) / 10;
-                o_accept_languages.append(&format!(";q=0.{}", u));
-            } else {
-                // Values below 10 require zero padding.
-                o_accept_languages.append(&format!(";q=0.{:02}", u));
-            }
+        //Since we need to emulate chrome behavior i.e languages should get q=1.0,0.9,0.8 and so on...
+        let q_val_max = 10;
+        let weight_of_decrement = 1;
+        let step = std::cmp::min(10, count_n); //if num_language > 10, q_val_max - curr_cnt*weight_of_decrement underflows
+        let q_val = std::cmp::max(q_val_max - step * weight_of_decrement, 1); //q-weight shouldn't go below 0.1
+
+        if count_n > 0 && q_val < 10 {
+            o_accept_languages.append(&format!(";q=0.{}", q_val));
         }
     }
 

netwerk/test/unit/test_header_Accept-Language.js

@@ -82,7 +82,7 @@ function test_accepted_languages() {
       // All the other languages should have an evenly-spaced quality value.
       Assert.equal(
         parseFloat(qualityValue).toFixed(decimalPlaces),
-        (1.0 - (1 / acceptedLanguagesLength) * i).toFixed(decimalPlaces)
+        Math.max(1.0 - i * 0.1, 0.1).toFixed(decimalPlaces)
       );
     }
   }

netwerk/test/unit/test_header_Accept-Language_case.js

@@ -18,11 +18,11 @@ function run_test() {
     ["ZH-HANT-HK", "zh-Hant-HK"],
     ["en-us-x-priv", "en-US-x-priv"],
     ["en-us-x-twain", "en-US-x-twain"],
-    ["de, en-US, en", "de,en-US;q=0.7,en;q=0.3"],
-    ["de,en-us,en", "de,en-US;q=0.7,en;q=0.3"],
-    ["en-US, en", "en-US,en;q=0.5"],
-    ["EN-US;q=0.2, EN", "en-US,en;q=0.5"],
-    ["en ;q=0.8, de  ", "en,de;q=0.5"],
+    ["de, en-US, en", "de,en-US;q=0.9,en;q=0.8"],
+    ["de,en-us,en", "de,en-US;q=0.9,en;q=0.8"],
+    ["en-US, en", "en-US,en;q=0.9"],
+    ["EN-US;q=0.2, EN", "en-US,en;q=0.9"],
+    ["en ;q=0.8, de  ", "en,de;q=0.9"],
     [",en,", "en"],
   ];