HTTPArchive
diff --git a/‎sql/2021/javascript/ajax_async.sql‎
Lines changed: 16 additions & 0 deletions b/‎sql/2021/javascript/ajax_async.sql‎
Lines changed: 16 additions & 0 deletions
diff --git a/‎sql/2021/javascript/ajax_request_per_page.sql‎
Lines changed: 2 additions & 0 deletions b/‎sql/2021/javascript/ajax_request_per_page.sql‎
Lines changed: 2 additions & 0 deletions
diff --git a/‎sql/2021/javascript/ajax_sync.sql‎
Lines changed: 16 additions & 0 deletions b/‎sql/2021/javascript/ajax_sync.sql‎
Lines changed: 16 additions & 0 deletions
diff --git a/‎sql/2021/javascript/breakdown_of_pages_using_async_defer.sql‎
Lines changed: 20 additions & 0 deletions b/‎sql/2021/javascript/breakdown_of_pages_using_async_defer.sql‎
Lines changed: 20 additions & 0 deletions
diff --git a/‎sql/2021/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql‎
Lines changed: 39 additions & 0 deletions b/‎sql/2021/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql‎
Lines changed: 39 additions & 0 deletions
diff --git a/‎sql/2021/javascript/lighthouse_vulnerabilities.sql‎
Lines changed: 18 additions & 0 deletions b/‎sql/2021/javascript/lighthouse_vulnerabilities.sql‎
Lines changed: 18 additions & 0 deletions
diff --git a/‎sql/2021/javascript/lighthouse_vulnerable_libraries.sql‎
Lines changed: 29 additions & 0 deletions b/‎sql/2021/javascript/lighthouse_vulnerable_libraries.sql‎
Lines changed: 29 additions & 0 deletions
diff --git a/‎sql/2021/javascript/module_and_nomodule.sql‎
Lines changed: 25 additions & 0 deletions b/‎sql/2021/javascript/module_and_nomodule.sql‎
Lines changed: 25 additions & 0 deletions
diff --git a/‎sql/2021/javascript/requests_by_3p.sql‎
Lines changed: 33 additions & 0 deletions b/‎sql/2021/javascript/requests_by_3p.sql‎
Lines changed: 33 additions & 0 deletions
diff --git a/‎sql/2021/javascript/web_components_specs.sql‎
Lines changed: 2 additions & 0 deletions b/‎sql/2021/javascript/web_components_specs.sql‎
Lines changed: 2 additions & 0 deletions
@@ -0,0 +1,16 @@
+# standardSQL
+# usage of async XMLHttpRequest using blink features from usage counters
+SELECT
+  client,
+  pct_urls
+FROM
+  `httparchive.blink_features.usage`
+WHERE
+  yyyymmdd = '20210701' AND
+  feature = 'XMLHttpRequestAsynchronous'
+GROUP BY
+  pct_urls,
+  client
+ORDER BY
+  pct_urls,
+  client
@@ -1,3 +1,5 @@
+# standardSQL
+# breadkdown of asynchronous JavaScript and XML requests per page
 SELECT
   client,
   percentile,
 
@@ -0,0 +1,16 @@
+# standardSQL
+# usage of synchronous XMLHttpRequest using blink features from usage counters
+SELECT
+  client,
+  pct_urls
+FROM
+  `httparchive.blink_features.usage`
+WHERE
+  yyyymmdd = '20210701' AND
+  feature = 'XMLHttpRequestSynchronous'
+GROUP BY
+  pct_urls,
+  client
+ORDER BY
+  pct_urls,
+  client
@@ -0,0 +1,20 @@
+#standardSQL
+# Breakdown of scripts using Async, Defer, Module or NoModule attributes.  Also breakdown of inline vs external scripts
+SELECT
+  _TABLE_SUFFIX AS client,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) AS async,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) AS defer,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) AS async_and_defer,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) AS module,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) AS nomodule,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) / COUNT(0) AS async_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) / COUNT(0) AS defer_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) / COUNT(0) AS async_and_defer_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) / COUNT(0) AS module_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) / COUNT(0) AS nomodule_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) AS neither,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct
+FROM
+  `httparchive.pages.2021_07_01_*`
+GROUP BY
+  client
@@ -0,0 +1,39 @@
+#standardSQL
+# Breakdown of scripts using Async, Defer, Module or NoModule attributes.  Also breakdown of inline vs external scripts
+CREATE TEMPORARY FUNCTION getScripts(payload STRING)
+RETURNS STRUCT<total INT64, inline INT64, src INT64, async INT64, defer INT64, async_and_defer INT64, type_module INT64, nomodule INT64>
+LANGUAGE js AS '''
+try {
+  var $ = JSON.parse(payload);
+  var javascript = JSON.parse($._javascript);
+  return javascript.script_tags;
+} catch (e) {
+  return {};
+}
+''';
+
+SELECT
+  client,
+  SUM(script.total) AS total_scripts,
+  SUM(script.inline) AS inline_script,
+  SUM(script.src) AS external_script,
+  SUM(script.src) / SUM(script.total) AS pct_external_script,
+  SUM(script.inline) / SUM(script.total) AS pct_inline_script,
+  SUM(script.async) AS async,
+  SUM(script.defer) AS defer,
+  SUM(script.async_and_defer) AS async_and_defer,
+  SUM(script.type_module) AS module,
+  SUM(script.nomodule) AS nomodule,
+  SUM(script.async) / SUM(script.src) AS pct_external_async,
+  SUM(script.defer) / SUM(script.src) AS pct_external_defer,
+  SUM(script.async_and_defer) / SUM(script.src) AS pct_external_async_defer,
+  SUM(script.type_module) / SUM(script.src) AS pct_external_module,
+  SUM(script.nomodule) / SUM(script.src) AS pct_external_nomodule
+FROM (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    getScripts(payload) AS script
+  FROM
+    `httparchive.pages.2021_07_01_*`)
+GROUP BY
+  client
@@ -0,0 +1,18 @@
+#standardSQL
+# Pages with vulnerable libraries
+SELECT
+  score,
+  COUNT(0) AS pages,
+  SUM(COUNT(0)) OVER () AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER () AS pct
+FROM (
+  SELECT
+    JSON_EXTRACT_SCALAR(report, "$.audits['no-vulnerable-libraries'].score") AS score
+  FROM
+    `httparchive.lighthouse.2021_07_01_mobile`)
+WHERE
+  score IS NOT NULL
+GROUP BY
+  score
+ORDER BY
+  score
@@ -0,0 +1,29 @@
+#standardSQL
+# Most frequent vulnerable libraries
+CREATE TEMPORARY FUNCTION getVulnerabilities(audit STRING)
+RETURNS ARRAY<STRING> LANGUAGE js AS '''
+try {
+  var $ = JSON.parse(audit);
+  return $.details.items.map(i => i.detectedLib.text.split('@')[0]);
+} catch(e) {
+  return [];
+}
+''';
+
+SELECT
+  lib,
+  COUNT(0) AS freq,
+  total,
+  COUNT(0) / total AS pct
+FROM
+  `httparchive.lighthouse.2021_07_01_mobile`,
+  UNNEST(getVulnerabilities(JSON_EXTRACT(report, "$.audits['no-vulnerable-libraries']"))) AS lib, (
+    SELECT
+      COUNT(DISTINCT url) AS total
+    FROM
+      `httparchive.lighthouse.2021_07_01_mobile`)
+GROUP BY
+  lib,
+  total
+ORDER BY
+  freq DESC
@@ -0,0 +1,25 @@
+SELECT
+  client,
+  COUNT(DISTINCT IF(module, page, NULL)) AS module,
+  COUNT(DISTINCT IF(nomodule, page, NULL)) AS nomodule,
+  COUNT(DISTINCT IF(nomodule AND module, page, NULL)) AS both,
+  COUNT(DISTINCT page) AS total,
+  COUNT(DISTINCT IF(module, page, NULL)) / COUNT(DISTINCT page) AS pct_module,
+  COUNT(DISTINCT IF(nomodule, page, NULL)) / COUNT(DISTINCT page) AS pct_nomodule,
+  COUNT(DISTINCT IF(module AND nomodule, page, NULL)) / COUNT(DISTINCT page) AS pct_both
+FROM (
+  SELECT
+    client,
+    page,
+    script,
+    REGEXP_CONTAINS(script, r'(?i)\bmodule\b') AS module,
+    REGEXP_CONTAINS(script, r'(?i)\bnomodule\b') AS nomodule
+  FROM
+    `httparchive.almanac.summary_response_bodies`
+  LEFT JOIN
+    UNNEST(REGEXP_EXTRACT_ALL(body, r'(?i)(<script[^>]*>)')) AS script
+  WHERE
+    date = '2021-07-01' AND
+    firstHtml)
+GROUP BY
+  client
@@ -0,0 +1,33 @@
+#standardSQL
+# Distribution of 1P/3P JS requests
+SELECT
+  percentile,
+  client,
+  host,
+  APPROX_QUANTILES(requests, 1000)[OFFSET(percentile * 10)] AS num_requests
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    COUNT(0) AS requests
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2021-07-01' AND
+    type = 'script'
+  GROUP BY
+    client,
+    page,
+    host),
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  host
+ORDER BY
+  percentile,
+  client,
+  host
@@ -11,6 +11,8 @@ FROM (
     ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.shadow_roots')) AS shadow_roots,
     ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.template')) AS template
   FROM
+    # Note: We're intentionally querying the September dataset here because of a bug in the custom metric.
+    # See https://github.com/HTTPArchive/legacy.httparchive.org/pull/231.
     `httparchive.pages.2021_09_01_*`),
   UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
 GROUP BY
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+# standardSQL`
	`2`	`+# breadkdown of asynchronous JavaScript and XML requests per page`
`1`	`3`	`SELECT`
`2`	`4`	`client,`
`3`	`5`	`percentile,`