From 5ec1e16efcd93165a5351d7ac79d6a400e1f2f43 Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Sun, 22 Oct 2023 08:38:50 -0400 Subject: [PATCH 1/7] cp 2022->2023 --- sql/2023/README.md | 50 +++ sql/2023/bfcache_cachecontrol_nostore.sql | 29 ++ sql/2023/bfcache_unload.sql | 31 ++ sql/2023/cls_animations.sql | 25 ++ sql/2023/cls_unsized_image_height.sql | 24 ++ sql/2023/cls_unsized_images.sql | 25 ++ sql/2023/correlation_preload_lcp.sql | 84 +++++ sql/2023/fid_long_tasks.sql | 35 ++ sql/2023/fid_tbt.sql | 13 + sql/2023/gaming_metric.sql | 25 ++ sql/2023/inp_long_tasks.sql | 63 ++++ sql/2023/js_bytes_rank.sql | 15 + sql/2023/lcp_bytes_distribution.sql | 36 ++ sql/2023/lcp_bytes_histogram.sql | 37 +++ sql/2023/lcp_element_data.sql | 133 ++++++++ sql/2023/lcp_element_data_2.sql | 104 ++++++ sql/2023/lcp_format.sql | 36 ++ sql/2023/lcp_host.sql | 26 ++ sql/2023/lcp_host_3p.sql | 28 ++ sql/2023/lcp_initiator_type.sql | 40 +++ sql/2023/lcp_lazy.sql | 46 +++ sql/2023/lcp_lazy_wordpress.sql | 63 ++++ sql/2023/lcp_preload.sql | 31 ++ sql/2023/lcp_preload_discoverable.sql | 25 ++ sql/2023/lcp_resource_delay.sql | 49 +++ sql/2023/lcp_resource_type.sql | 28 ++ sql/2023/lcp_responsive_data.sql | 34 ++ sql/2023/lcp_wasted_bytes.sql | 56 ++++ sql/2023/monthly_cls_lcp.sql | 16 + sql/2023/prelcp_domain_sharding.sql | 58 ++++ sql/2023/render_blocking_resources.sql | 22 ++ sql/2023/ttfb_by_cms.sql | 26 ++ sql/2023/ttfb_by_rendering.sql | 20 ++ sql/2023/viewport_meta_zoom_disable.sql | 6 + sql/2023/web_vitals_by_country.sql | 176 ++++++++++ sql/2023/web_vitals_by_device.sql | 175 ++++++++++ .../web_vitals_by_eff_connection_type.sql | 309 ++++++++++++++++++ sql/2023/web_vitals_by_rank.sql | 182 +++++++++++ sql/2023/web_vitals_by_rank_and_device.sql | 193 +++++++++++ sql/2023/web_vitals_by_technology.sql | 208 ++++++++++++ 40 files changed, 2582 insertions(+) create mode 100644 sql/2023/README.md create mode 100644 sql/2023/bfcache_cachecontrol_nostore.sql create mode 100644 sql/2023/bfcache_unload.sql create mode 100644 sql/2023/cls_animations.sql create mode 100644 sql/2023/cls_unsized_image_height.sql create mode 100644 sql/2023/cls_unsized_images.sql create mode 100644 sql/2023/correlation_preload_lcp.sql create mode 100644 sql/2023/fid_long_tasks.sql create mode 100644 sql/2023/fid_tbt.sql create mode 100644 sql/2023/gaming_metric.sql create mode 100644 sql/2023/inp_long_tasks.sql create mode 100644 sql/2023/js_bytes_rank.sql create mode 100644 sql/2023/lcp_bytes_distribution.sql create mode 100644 sql/2023/lcp_bytes_histogram.sql create mode 100644 sql/2023/lcp_element_data.sql create mode 100644 sql/2023/lcp_element_data_2.sql create mode 100644 sql/2023/lcp_format.sql create mode 100644 sql/2023/lcp_host.sql create mode 100644 sql/2023/lcp_host_3p.sql create mode 100644 sql/2023/lcp_initiator_type.sql create mode 100644 sql/2023/lcp_lazy.sql create mode 100644 sql/2023/lcp_lazy_wordpress.sql create mode 100644 sql/2023/lcp_preload.sql create mode 100644 sql/2023/lcp_preload_discoverable.sql create mode 100644 sql/2023/lcp_resource_delay.sql create mode 100644 sql/2023/lcp_resource_type.sql create mode 100644 sql/2023/lcp_responsive_data.sql create mode 100644 sql/2023/lcp_wasted_bytes.sql create mode 100644 sql/2023/monthly_cls_lcp.sql create mode 100644 sql/2023/prelcp_domain_sharding.sql create mode 100644 sql/2023/render_blocking_resources.sql create mode 100644 sql/2023/ttfb_by_cms.sql create mode 100644 sql/2023/ttfb_by_rendering.sql create mode 100644 sql/2023/viewport_meta_zoom_disable.sql create mode 100644 sql/2023/web_vitals_by_country.sql create mode 100644 sql/2023/web_vitals_by_device.sql create mode 100644 sql/2023/web_vitals_by_eff_connection_type.sql create mode 100644 sql/2023/web_vitals_by_rank.sql create mode 100644 sql/2023/web_vitals_by_rank_and_device.sql create mode 100644 sql/2023/web_vitals_by_technology.sql diff --git a/sql/2023/README.md b/sql/2023/README.md new file mode 100644 index 00000000000..fea835c8619 --- /dev/null +++ b/sql/2023/README.md @@ -0,0 +1,50 @@ +# 2022 Performance queries + + + +## Resources + +- [📄 Planning doc][~google-doc] +- [📊 Results sheet][~google-sheets] +- [📝 Markdown file][~chapter-markdown] + +[~google-doc]: https://docs.google.com/document/d/1IKV40fllCZTqeu-R6-73ckjQR9S6jiBfVBBfdcpAMkI/edit?usp=sharing +[~google-sheets]: https://docs.google.com/spreadsheets/d/1TPA_4xRTBB2fQZaBPZHVFvD0ikrR-4sNkfJfUEpjibs/edit?usp=sharing +[~chapter-markdown]: https://github.com/HTTPArchive/almanac.httparchive.org/tree/main/src/content/en/2022/performance.md + +## Query List + +Taken from [`Metrics` Section](https://docs.google.com/document/d/1IKV40fllCZTqeu-R6-73ckjQR9S6jiBfVBBfdcpAMkI/edit#heading=h.zbvh8yhwkp2i) in planning doc, where technical notes on how queries might be formulated live. + +## Query notes + +### `inp_long_tasks.sql` + +This query generates a scatterplot of field-based, page-level p75 INP versus the sum of all lab-based long tasks on the page according to Lighthouse. + +Lab-based mobile CPU throttling necessarily means that long tasks should be higher than desktop. It's interesting to look at the trendlines for both desktop and mobile to see that higher INP correlates with higher long tasks. + +### `lcp_resource_delay.sql` + +This query subtracts the lab-based TTFB time from the lab-based LCP element's request start time and generates a distribution over all pages. + +The `lcp_elem_stats.startTime` and `lcp_resource.timestamp` values did not seem to correspond to the actual LCP element request start times seen in the WPT results, so the query goes the more expensive route to join the pages data with the more reliable requests data. + +### `prelcp_domain_sharding.sql` + +This query takes the distribution of the number of unique hosts connected prior to the lab-based LCP time, broken down by whether the LCP was good, NI, or poor. + +### `ttfb_by_rendering.sql` + +This query segments pages by whether they use client-side rendering (CSR) or server-side rendering (SSR). There is no high quality signal for CSR/SSR so the heuristic used in this query is the ratio of the number of words in the document served in the static HTML to the number of words in the final rendered page. If the number of words increases by 1.5x after rendering, then we consider the page to use CSR. + +Using both rendering buckets, we then calculate the median field-based p75 TTFB. + +This metric is not expected to be very meaningful given the tenuous definition of CSR. diff --git a/sql/2023/bfcache_cachecontrol_nostore.sql b/sql/2023/bfcache_cachecontrol_nostore.sql new file mode 100644 index 00000000000..71116136aa2 --- /dev/null +++ b/sql/2023/bfcache_cachecontrol_nostore.sql @@ -0,0 +1,29 @@ +CREATE TEMP FUNCTION HAS_NO_STORE_DIRECTIVE(cache_control STRING) RETURNS BOOL AS ( + REGEXP_CONTAINS(cache_control, r'(?i)\bno-store\b') +); + +WITH requests AS ( + SELECT + client, + LOGICAL_OR(HAS_NO_STORE_DIRECTIVE(JSON_VALUE(payload, '$._cacheControl'))) AS includes_ccns + FROM + `httparchive.all.requests` + WHERE + date = '2023-10-01' AND + is_main_document + GROUP BY + client, + page +) + +SELECT + client, + COUNTIF(includes_ccns) AS pages, + COUNT(0) AS total, + COUNTIF(includes_ccns) / COUNT(0) AS pct +FROM + requests +GROUP BY + client +ORDER BY + client diff --git a/sql/2023/bfcache_unload.sql b/sql/2023/bfcache_unload.sql new file mode 100644 index 00000000000..8aecd755ba1 --- /dev/null +++ b/sql/2023/bfcache_unload.sql @@ -0,0 +1,31 @@ +WITH lh AS ( + SELECT + client, + page, + rank, + JSON_VALUE(lighthouse, '$.audits.no-unload-listeners.score') = '0' AS has_unload + FROM + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page +) + + +SELECT + client, + _rank AS rank, + COUNTIF(has_unload) AS pages, + COUNT(0) AS total, + COUNTIF(has_unload) / COUNT(0) AS pct +FROM + lh, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS _rank +WHERE + rank <= _rank +GROUP BY + client, + rank +ORDER BY + rank, + client diff --git a/sql/2023/cls_animations.sql b/sql/2023/cls_animations.sql new file mode 100644 index 00000000000..1452e404fc4 --- /dev/null +++ b/sql/2023/cls_animations.sql @@ -0,0 +1,25 @@ +WITH lh AS ( + SELECT + _TABLE_SUFFIX AS client, + ARRAY_LENGTH(JSON_QUERY_ARRAY(report, '$.audits.non-composited-animations.details.items')) AS num_animations + FROM + `httparchive.lighthouse.2022_06_01_*` +) + + +SELECT + percentile, + client, + APPROX_QUANTILES(num_animations, 1000)[OFFSET(percentile * 10)] AS num_animations, + COUNTIF(num_animations > 0) AS pages, + COUNT(0) AS total, + COUNTIF(num_animations > 0) / COUNT(0) AS pct +FROM + lh, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/cls_unsized_image_height.sql b/sql/2023/cls_unsized_image_height.sql new file mode 100644 index 00000000000..2b853dfa73a --- /dev/null +++ b/sql/2023/cls_unsized_image_height.sql @@ -0,0 +1,24 @@ +WITH lh AS ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_VALUE(unsized_image, '$.node.boundingRect.height') AS INT64) AS height + FROM + `httparchive.lighthouse.2022_06_01_*`, + UNNEST(JSON_QUERY_ARRAY(report, '$.audits.unsized-images.details.items')) AS unsized_image +) + + +SELECT + percentile, + client, + APPROX_QUANTILES(height, 1000)[OFFSET(percentile * 10)] AS height, + COUNT(0) AS unsized_images +FROM + lh, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/cls_unsized_images.sql b/sql/2023/cls_unsized_images.sql new file mode 100644 index 00000000000..5a5bd4c2e39 --- /dev/null +++ b/sql/2023/cls_unsized_images.sql @@ -0,0 +1,25 @@ +WITH lh AS ( + SELECT + _TABLE_SUFFIX AS client, + ARRAY_LENGTH(JSON_QUERY_ARRAY(report, '$.audits.unsized-images.details.items')) AS num_unsized_images + FROM + `httparchive.lighthouse.2022_06_01_*` +) + + +SELECT + percentile, + client, + APPROX_QUANTILES(num_unsized_images, 1000)[OFFSET(percentile * 10)] AS num_unsized_images, + COUNTIF(num_unsized_images > 0) AS pages, + COUNT(0) AS total, + COUNTIF(num_unsized_images > 0) / COUNT(0) AS pct +FROM + lh, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/correlation_preload_lcp.sql b/sql/2023/correlation_preload_lcp.sql new file mode 100644 index 00000000000..ce5329028b0 --- /dev/null +++ b/sql/2023/correlation_preload_lcp.sql @@ -0,0 +1,84 @@ +CREATE TEMPORARY FUNCTION getResourceHints(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +var hints = ['preload']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return hints.reduce((results, hint) => { + // Null values are omitted from BigQuery aggregations. + // This means only pages with at least one hint are considered. + results[hint] = almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).length || 0; + return results; + }, {}); +} catch (e) { + return hints.reduce((results, hint) => { + results[hint] = 0; + return results; + }, {}); +} +'''; + +CREATE TEMPORARY FUNCTION getGoodCwv(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var crux = $._CrUX; + + if (crux) { + return Object.keys(crux.metrics).reduce((acc, n) => ({ + ...acc, + [n]: crux.metrics[n].histogram[0].density > 0.75 + }), {}) + } + + return null; +} catch (e) { + return null; +} +'''; + +SELECT + device, + + LEAST(hints.preload, 30) AS preload, + + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY device) AS total, + + COUNTIF(CrUX.largest_contentful_paint) AS lcp_good, + COUNTIF(CrUX.largest_contentful_paint IS NOT NULL) AS any_lcp, + COUNTIF(CrUX.largest_contentful_paint) / COUNTIF(CrUX.largest_contentful_paint IS NOT NULL) AS pct_lcp_good, + + COUNTIF(CrUX.first_input_delay) AS fid_good, + COUNTIF(CrUX.first_input_delay IS NOT NULL) AS any_fid, + COUNTIF(CrUX.first_input_delay) / COUNTIF(CrUX.first_input_delay IS NOT NULL) AS pct_fid_good, + + COUNTIF(CrUX.cumulative_layout_shift) AS cls_good, + COUNTIF(CrUX.cumulative_layout_shift IS NOT NULL) AS any_cls, + COUNTIF(CrUX.cumulative_layout_shift) / COUNTIF(CrUX.cumulative_layout_shift IS NOT NULL) AS pct_cls_good, + + COUNTIF(CrUX.first_contentful_paint) AS fcp_good, + COUNTIF(CrUX.first_contentful_paint IS NOT NULL) AS any_fcp, + COUNTIF(CrUX.first_contentful_paint) / COUNTIF(CrUX.first_contentful_paint IS NOT NULL) AS pct_fcp_good, + + COUNTIF(CrUX.largest_contentful_paint AND CrUX.first_input_delay IS NOT FALSE AND CrUX.cumulative_layout_shift) AS cwv_good, + COUNTIF(CrUX.largest_contentful_paint IS NOT NULL AND CrUX.cumulative_layout_shift IS NOT NULL) AS eligible_cwv, + COUNTIF(CrUX.largest_contentful_paint AND CrUX.first_input_delay IS NOT FALSE AND CrUX.cumulative_layout_shift) / COUNTIF(CrUX.largest_contentful_paint IS NOT NULL AND CrUX.cumulative_layout_shift IS NOT NULL) AS pct_cwv_good +FROM ( + SELECT + _TABLE_SUFFIX AS device, + getResourceHints(payload) AS hints, + getGoodCwv(payload) AS CrUX + FROM + `httparchive.pages.2022_06_01_*` +) +WHERE + CrUX IS NOT NULL +GROUP BY + device, + preload +ORDER BY + device, + preload diff --git a/sql/2023/fid_long_tasks.sql b/sql/2023/fid_long_tasks.sql new file mode 100644 index 00000000000..6fec7d7bb5b --- /dev/null +++ b/sql/2023/fid_long_tasks.sql @@ -0,0 +1,35 @@ +WITH long_tasks AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + CAST(JSON_QUERY(item, '$.duration') AS FLOAT64) AS long_task_duration + FROM + `lighthouse.2022_06_01_*`, + UNNEST(JSON_QUERY_ARRAY(report, '$.audits.long-tasks.details.items')) AS item +), + +per_page AS ( + SELECT + client, + page, + SUM(long_task_duration) AS long_tasks + FROM + long_tasks + GROUP BY + client, + page +) + +SELECT + percentile, + client, + APPROX_QUANTILES(long_tasks, 1000)[OFFSET(percentile * 10)] AS long_tasks +FROM + per_page, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/fid_tbt.sql b/sql/2023/fid_tbt.sql new file mode 100644 index 00000000000..a4bf92f8dff --- /dev/null +++ b/sql/2023/fid_tbt.sql @@ -0,0 +1,13 @@ +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(CAST(JSON_QUERY(report, '$.audits.total-blocking-time.numericValue') AS FLOAT64), 1000)[OFFSET(percentile * 10)] AS tbt +FROM + `httparchive.lighthouse.2022_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/gaming_metric.sql b/sql/2023/gaming_metric.sql new file mode 100644 index 00000000000..567430c4145 --- /dev/null +++ b/sql/2023/gaming_metric.sql @@ -0,0 +1,25 @@ +SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-ChromeLH') = 'true', 1, 0)) AS chrome_lighthouse, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-ChromeLH') = 'true', 1, 0)) / COUNT(0) AS chrome_lighthouse_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-GTmetrix') = 'true', 1, 0)) AS gtmetrix, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-GTmetrix') = 'true', 1, 0)) / COUNT(0) AS gtmetrix_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-PageSpeed') = 'true', 1, 0)) AS pagespeed, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-PageSpeed') = 'true', 1, 0)) / COUNT(0) AS pagespeed_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationStrict') = 'true', 1, 0)) AS img_animation_strict, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationStrict') = 'true', 1, 0)) / COUNT(0) AS img_animation_strict_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationSoft') = 'true', 1, 0)) AS img_animation_soft, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationSoft') = 'true', 1, 0)) / COUNT(0) AS img_animation_soft_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlayStrict') = 'true', 1, 0)) AS fid_iframe_overlay_strict, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlayStrict') = 'true', 1, 0)) / COUNT(0) AS fid_iframe_overlay_strict_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlaySoft') = 'true', 1, 0)) AS fid_iframe_overlay_soft, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlaySoft') = 'true', 1, 0)) / COUNT(0) AS fid_iframe_overlay_soft_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlayStrict') = 'true', 1, 0)) AS lcp_overlay_strict, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlayStrict') = 'true', 1, 0)) / COUNT(0) AS lcp_overlay_strict_per, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlaySoft') = 'true', 1, 0)) AS lcp_overlay_soft, + SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlaySoft') = 'true', 1, 0)) / COUNT(0) AS lcp_overlay_soft_per +FROM + `httparchive.pages.2022_06_01_*` +GROUP BY + client diff --git a/sql/2023/inp_long_tasks.sql b/sql/2023/inp_long_tasks.sql new file mode 100644 index 00000000000..22f9f4cb98e --- /dev/null +++ b/sql/2023/inp_long_tasks.sql @@ -0,0 +1,63 @@ +WITH long_tasks AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + CAST(JSON_QUERY(item, '$.duration') AS FLOAT64) AS long_task_duration + FROM + `lighthouse.2022_06_01_*`, + UNNEST(JSON_QUERY_ARRAY(report, '$.audits.long-tasks.details.items')) AS item +), + +per_page AS ( + SELECT + client, + page, + SUM(long_task_duration) AS long_tasks + FROM + long_tasks + GROUP BY + client, + page +), + +crux_inp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + httparchive.core_web_vitals.GET_CRUX_INP(payload) AS inp + FROM + `httparchive.pages.2022_06_01_*` +), + +combined AS ( + SELECT + client, + long_tasks, + inp + FROM + per_page + JOIN + crux_inp + USING + (client, page) +), + +meta AS ( + SELECT + *, + COUNT(0) OVER (PARTITION BY client) AS n, + ROW_NUMBER() OVER (PARTITION BY client ORDER BY inp) AS row + FROM + combined + WHERE + inp IS NOT NULL +) + +SELECT + client, + long_tasks, + inp +FROM + meta +WHERE + MOD(row, CAST(FLOOR(n / 1000) AS INT64)) = 0 diff --git a/sql/2023/js_bytes_rank.sql b/sql/2023/js_bytes_rank.sql new file mode 100644 index 00000000000..ae0f3d25e82 --- /dev/null +++ b/sql/2023/js_bytes_rank.sql @@ -0,0 +1,15 @@ +SELECT + IF(_rank < 10000000, CAST(_rank AS STRING), 'all') AS rank, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJS, 1001)[OFFSET(501)] / 1024 AS js_kbytes +FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS _rank +WHERE + rank <= _rank +GROUP BY + rank, + client +ORDER BY + rank, + client diff --git a/sql/2023/lcp_bytes_distribution.sql b/sql/2023/lcp_bytes_distribution.sql new file mode 100644 index 00000000000..7ed7e0dcd70 --- /dev/null +++ b/sql/2023/lcp_bytes_distribution.sql @@ -0,0 +1,36 @@ +WITH pages AS ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + FROM + `httparchive.pages.2022_06_01_*` +), + +requests AS ( + SELECT + _TABLE_SUFFIX AS client, + pageid, + url, + respSize / 1024 AS kbytes + FROM + `httparchive.summary_requests.2022_06_01_*` +) + +SELECT + percentile, + client, + APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes +FROM + pages +JOIN + requests +USING + (client, pageid, url), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/lcp_bytes_histogram.sql b/sql/2023/lcp_bytes_histogram.sql new file mode 100644 index 00000000000..49c3ec54e58 --- /dev/null +++ b/sql/2023/lcp_bytes_histogram.sql @@ -0,0 +1,37 @@ +WITH pages AS ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + FROM + `httparchive.pages.2022_06_01_*` +), + +requests AS ( + SELECT + _TABLE_SUFFIX AS client, + pageid, + url, + respSize / 1024 AS kbytes + FROM + `httparchive.summary_requests.2022_06_01_*` +) + +SELECT + client, + IF(CEILING(kbytes / 100) * 100 < 1000, CAST(CEILING(kbytes / 100) * 100 AS STRING), '1000+') AS kbytes, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + pages +JOIN + requests +USING + (client, pageid, url) +GROUP BY + client, + kbytes +ORDER BY + client, + kbytes diff --git a/sql/2023/lcp_element_data.sql b/sql/2023/lcp_element_data.sql new file mode 100644 index 00000000000..a3d8460c60e --- /dev/null +++ b/sql/2023/lcp_element_data.sql @@ -0,0 +1,133 @@ +#standardSQL +# LCP element node details + +CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const loadingAttr = data.find(attr => attr["name"] === "loading") + return loadingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION getDecodingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const decodingAttr = data.find(attr => attr["name"] === "decoding") + return decodingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION getLoadingClasses(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const classes = data.find(attr => attr["name"] === "class").value + if (classes.indexOf('lazyload') !== -1) { + return classes + } else { + return "" + } + } catch (e) { + return ""; + } +'''; + +CREATE TEMPORARY FUNCTION getResourceHints(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +var hints = ['preload', 'prefetch', 'preconnect', 'prerender', 'dns-prefetch', 'modulepreload']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return hints.reduce((results, hint) => { + results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint); + return results; + }, {}); +} catch (e) { + return hints.reduce((results, hint) => { + results[hint] = false; + return results; + }, {}); +} +'''; + +CREATE TEMPORARY FUNCTION getFetchPriority(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +var hints = ['high', 'low', 'auto']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return hints.reduce((results, hint) => { + results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint); + return results; + }, {}); +} catch (e) { + return hints.reduce((results, hint) => { + results[hint] = false; + return results; + }, {}); +} +'''; + +WITH +lcp_stats AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') AS nodeName, + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.url') AS elementUrl, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.size') AS INT64) AS size, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.loadTime') AS FLOAT64) AS loadTime, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.startTime') AS FLOAT64) AS startTime, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.renderTime') AS FLOAT64) AS renderTime, + JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes') AS attributes, + getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS loading, + getDecodingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS decoding, + getLoadingClasses(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS classWithLazyload, + getResourceHints(payload) AS hints + FROM + `httparchive.pages.2022_06_01_*` +) + +SELECT + client, + nodeName, + COUNT(DISTINCT url) AS pages, + ANY_VALUE(total) AS total, + COUNT(DISTINCT url) / ANY_VALUE(total) AS pct, + COUNTIF(elementUrl != '') AS haveImages, + COUNTIF(elementUrl != '') / COUNT(DISTINCT url) AS pct_haveImages, + COUNTIF(loading = 'eager') AS native_eagerload, + COUNTIF(loading = 'lazy') AS native_lazyload, + COUNTIF(classWithLazyload != '') AS lazyload_class, + COUNTIF(classWithLazyload != '' OR loading = 'lazy') AS probably_lazyLoaded, + COUNTIF(classWithLazyload != '' OR loading = 'lazy') / COUNT(DISTINCT url) AS pct_prob_lazyloaded, + COUNTIF(decoding = 'async') AS async_decoding, + COUNTIF(decoding = 'sync') AS sync_decoding, + COUNTIF(decoding = 'auto') AS auto_decoding, + COUNT(0) AS total, + COUNTIF(hints.preload) AS preload, + COUNTIF(hints.preload) / COUNT(0) AS pct_preload +FROM + lcp_stats +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (client) +GROUP BY + client, + nodeName +HAVING + pages > 1000 +ORDER BY + pct DESC diff --git a/sql/2023/lcp_element_data_2.sql b/sql/2023/lcp_element_data_2.sql new file mode 100644 index 00000000000..d5d2e4207ec --- /dev/null +++ b/sql/2023/lcp_element_data_2.sql @@ -0,0 +1,104 @@ +#standardSQL +# LCP element node details + +CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const loadingAttr = data.find(attr => attr["name"] === "loading") + return loadingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION getDecodingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const decodingAttr = data.find(attr => attr["name"] === "decoding") + return decodingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION getFetchPriorityAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const fetchPriorityAttr = data.find(attr => attr["name"] === "fetchpriority") + return fetchPriorityAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION getLoadingClasses(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const classes = data.find(attr => attr["name"] === "class").value + if (classes.indexOf('lazyload') !== -1) { + return classes + } else { + return "" + } + } catch (e) { + return ""; + } +'''; + +WITH +lcp_stats AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') AS nodeName, + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.url') AS elementUrl, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.size') AS INT64) AS size, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.loadTime') AS FLOAT64) AS loadTime, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.startTime') AS FLOAT64) AS startTime, + CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.renderTime') AS FLOAT64) AS renderTime, + JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes') AS attributes, + getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS loading, + getDecodingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS decoding, + getLoadingClasses(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS classWithLazyload, + getFetchPriorityAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS fetchPriority + FROM + `httparchive.pages.2022_06_01_*` +) + +SELECT + client, + nodeName, + COUNT(DISTINCT url) AS pages, + ANY_VALUE(total) AS total, + COUNT(DISTINCT url) / ANY_VALUE(total) AS pct, + COUNTIF(elementUrl != '') AS haveImages, + COUNTIF(elementUrl != '') / COUNT(DISTINCT url) AS pct_haveImages, + COUNTIF(loading = 'eager') AS native_eagerload, + COUNTIF(loading = 'lazy') AS native_lazyload, + COUNTIF(classWithLazyload != '') AS lazyload_class, + COUNTIF(classWithLazyload != '' OR loading = 'lazy') AS probably_lazyLoaded, + COUNTIF(classWithLazyload != '' OR loading = 'lazy') / COUNT(DISTINCT url) AS pct_prob_lazyloaded, + COUNTIF(decoding = 'async') AS async_decoding, + COUNTIF(decoding = 'sync') AS sync_decoding, + COUNTIF(decoding = 'auto') AS auto_decoding, + COUNTIF(fetchPriority = 'low') AS priority_low, + COUNTIF(fetchPriority = 'high') AS priority_high +FROM + lcp_stats +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total + FROM + `httparchive.pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (client) +GROUP BY + client, + nodeName +HAVING + pages > 1000 +ORDER BY + pct DESC diff --git a/sql/2023/lcp_format.sql b/sql/2023/lcp_format.sql new file mode 100644 index 00000000000..7bf15f18d6c --- /dev/null +++ b/sql/2023/lcp_format.sql @@ -0,0 +1,36 @@ +WITH pages AS ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + FROM + `httparchive.pages.2022_06_01_*` +), + +requests AS ( + SELECT + _TABLE_SUFFIX AS client, + pageid, + url, + format + FROM + `httparchive.summary_requests.2022_06_01_*` +) + +SELECT + client, + format, + COUNT(0) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + pages +JOIN + requests +USING + (client, pageid, url) +GROUP BY + client, + format +ORDER BY + pct DESC diff --git a/sql/2023/lcp_host.sql b/sql/2023/lcp_host.sql new file mode 100644 index 00000000000..528fec937c8 --- /dev/null +++ b/sql/2023/lcp_host.sql @@ -0,0 +1,26 @@ +WITH lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + FROM + `httparchive.pages.2022_06_01_*` +) + + +SELECT + client, + CASE + WHEN NET.HOST(url) = 'data' THEN 'other content' + WHEN NET.HOST(url) IS NULL THEN 'other content' + WHEN NET.HOST(page) = NET.HOST(url) THEN 'same host' + ELSE 'cross host' + END AS lcp_same_host, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + lcp +GROUP BY + client, + lcp_same_host diff --git a/sql/2023/lcp_host_3p.sql b/sql/2023/lcp_host_3p.sql new file mode 100644 index 00000000000..4fda76c04e9 --- /dev/null +++ b/sql/2023/lcp_host_3p.sql @@ -0,0 +1,28 @@ +WITH lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + FROM + `httparchive.pages.2022_06_01_*` +) + + +SELECT + client, + NET.REG_DOMAIN(url) AS lcp_domain, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + lcp +WHERE + NET.HOST(page) != NET.HOST(url) AND + NET.HOST(url) != 'data' +GROUP BY + client, + lcp_domain +ORDER BY + pct DESC +LIMIT + 25 diff --git a/sql/2023/lcp_initiator_type.sql b/sql/2023/lcp_initiator_type.sql new file mode 100644 index 00000000000..dbb611ffb3b --- /dev/null +++ b/sql/2023/lcp_initiator_type.sql @@ -0,0 +1,40 @@ +WITH lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + JSON_VALUE(payload, '$._performance.lcp_resource.initiator.url') AS url, + JSON_VALUE(payload, '$._performance.is_lcp_statically_discoverable') = 'false' AS not_discoverable + FROM + `httparchive.pages.2022_06_01_*` +), + +requests AS ( + SELECT + client, + page, + url, + type + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' +) + + +SELECT + client, + type AS lcp_initiator_type, + COUNTIF(not_discoverable) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNTIF(not_discoverable) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + lcp +JOIN + requests +USING + (client, page, url) +GROUP BY + client, + type +ORDER BY + pct DESC diff --git a/sql/2023/lcp_lazy.sql b/sql/2023/lcp_lazy.sql new file mode 100644 index 00000000000..23fd7540827 --- /dev/null +++ b/sql/2023/lcp_lazy.sql @@ -0,0 +1,46 @@ +CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const loadingAttr = data.find(attr => attr["name"] === "loading") + return loadingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION hasLazyHeuristics(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const classes = data.find(attr => attr["name"] === "class").value; + const hasLazyClasses = classes.indexOf('lazyload') !== -1; + const hasLazySrc = data.includes(attr => attr["name"] === "data-src"); + + return hasLazyClasses || hasLazySrc; + } catch (e) { + return false; + } +'''; + +WITH lcp_stats AS ( + SELECT + _TABLE_SUFFIX AS client, + getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) = 'lazy' AS native_lazy, + hasLazyHeuristics(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS custom_lazy + FROM + `httparchive.pages.2022_06_01_*` + WHERE + # LCP only. + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') = 'IMG' +) + +SELECT + client, + COUNT(0) AS total, + COUNTIF(native_lazy) / COUNT(0) AS pct_native_lazy, + COUNTIF(custom_lazy) / COUNT(0) AS pct_custom_lazy, + COUNTIF(custom_lazy OR native_lazy) / COUNT(0) AS pct_either_lazy, + COUNTIF(custom_lazy AND native_lazy) / COUNT(0) AS pct_both_lazy +FROM + lcp_stats +GROUP BY + client diff --git a/sql/2023/lcp_lazy_wordpress.sql b/sql/2023/lcp_lazy_wordpress.sql new file mode 100644 index 00000000000..9194ec592af --- /dev/null +++ b/sql/2023/lcp_lazy_wordpress.sql @@ -0,0 +1,63 @@ +CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const loadingAttr = data.find(attr => attr["name"] === "loading") + return loadingAttr.value + } catch (e) { + return ""; + } +'''; + +CREATE TEMP FUNCTION hasLazyHeuristics(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const classes = data.find(attr => attr["name"] === "class").value; + const hasLazyClasses = classes.indexOf('lazyload') !== -1; + const hasLazySrc = data.includes(attr => attr["name"] === "data-src"); + + return hasLazyClasses || hasLazySrc; + } catch (e) { + return false; + } +'''; + +WITH lazy AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) = 'lazy' AS native_lazy, + hasLazyHeuristics(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS custom_lazy + FROM + `httparchive.pages.2022_06_01_*` + WHERE + # LCP only. + JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') = 'IMG' +), + +wp AS ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url, + TRUE AS wordpress + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + app = 'WordPress' +) + + +SELECT + client, + COUNTIF(wordpress) AS wordpress, + COUNTIF(native_lazy) AS native_lazy, + COUNTIF(custom_lazy) AS custom_lazy, + COUNTIF(wordpress AND native_lazy) / COUNTIF(native_lazy) AS pct_wordpress_native_lazy, + COUNTIF(wordpress AND custom_lazy) / COUNTIF(custom_lazy) AS pct_wordpress_custom_lazy +FROM + lazy +LEFT JOIN + wp +USING + (client, url) +GROUP BY + client diff --git a/sql/2023/lcp_preload.sql b/sql/2023/lcp_preload.sql new file mode 100644 index 00000000000..b814e52c4b9 --- /dev/null +++ b/sql/2023/lcp_preload.sql @@ -0,0 +1,31 @@ +CREATE TEMPORARY FUNCTION isLCPPreloaded(payload STRING) RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var lcp_url = $._performance.lcp_elem_stats.url; + + var almanac = JSON.parse($._almanac); + return !!almanac['link-nodes'].nodes.find(link => { + return link.rel && link.rel.toLowerCase() == 'preload' && lcp_url.endsWith(link.href); + }); +} catch (e) { + return false; +} +'''; + +WITH preloaded AS ( + SELECT + _TABLE_SUFFIX AS client, + isLCPPreloaded(payload) AS is_lcp_preloaded + FROM + `httparchive.pages.2022_06_01_*` +) + +SELECT + client, + COUNTIF(is_lcp_preloaded) AS lcp_preloaded, + COUNT(0) AS total, + COUNTIF(is_lcp_preloaded) / COUNT(0) AS pct_lcp_preloaded +FROM + preloaded +GROUP BY + client diff --git a/sql/2023/lcp_preload_discoverable.sql b/sql/2023/lcp_preload_discoverable.sql new file mode 100644 index 00000000000..8e800a7c993 --- /dev/null +++ b/sql/2023/lcp_preload_discoverable.sql @@ -0,0 +1,25 @@ +WITH lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + JSON_VALUE(payload, '$._performance.is_lcp_statically_discoverable') = 'true' AS discoverable, + JSON_VALUE(payload, '$._performance.is_lcp_preloaded') = 'true' AS preloaded + FROM + `httparchive.pages.2022_06_01_*` +) + + +SELECT + client, + discoverable, + preloaded, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + lcp +GROUP BY + client, + discoverable, + preloaded +ORDER BY + pct DESC diff --git a/sql/2023/lcp_resource_delay.sql b/sql/2023/lcp_resource_delay.sql new file mode 100644 index 00000000000..c034510b258 --- /dev/null +++ b/sql/2023/lcp_resource_delay.sql @@ -0,0 +1,49 @@ +WITH pages AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url, + httparchive.core_web_vitals.GET_LAB_TTFB(payload) AS ttfb + FROM + `httparchive.pages.2022_06_01_*` +), + +requests AS ( + SELECT + _TABLE_SUFFIX AS client, + page, + url, + CAST(JSON_QUERY(payload, '$._created') AS FLOAT64) AS lcp_req_time + FROM + `httparchive.requests.2022_06_01_*` +), + +delays AS ( + SELECT + client, + CAST(lcp_req_time - ttfb AS INT64) AS lcp_resource_delay + FROM + pages + JOIN + requests + USING + (client, page, url) + WHERE + lcp_req_time IS NOT NULL AND + lcp_req_time > 0 AND + ttfb IS NOT NULL +) + +SELECT + percentile, + client, + APPROX_QUANTILES(lcp_resource_delay, 1000)[OFFSET(percentile * 10)] AS lcp_resource_delay +FROM + delays, + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/lcp_resource_type.sql b/sql/2023/lcp_resource_type.sql new file mode 100644 index 00000000000..fae34d9e39f --- /dev/null +++ b/sql/2023/lcp_resource_type.sql @@ -0,0 +1,28 @@ +WITH lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + # Parse anchors out of LCP URLs. + REGEXP_EXTRACT(JSON_VALUE(payload, '$._performance.lcp_elem_stats.url'), r'([^#]*)') AS url + FROM + `httparchive.pages.2022_06_01_*` +) + + +SELECT + client, + CASE + WHEN lcp.url = '' THEN 'text' + WHEN STARTS_WITH(lcp.url, 'data:') THEN 'inline image' + ELSE 'image' + END AS lcp_type, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + lcp +GROUP BY + client, + lcp_type +ORDER BY + pct DESC diff --git a/sql/2023/lcp_responsive_data.sql b/sql/2023/lcp_responsive_data.sql new file mode 100644 index 00000000000..8077dcb37b5 --- /dev/null +++ b/sql/2023/lcp_responsive_data.sql @@ -0,0 +1,34 @@ +CREATE TEMP FUNCTION checkResponsiveImages(responsivelist STRING, lcpImgUrl STRING, nodePath STRING) RETURNS BOOLEAN LANGUAGE js AS ''' + try { + //we will check lcp elment is img + const lastSegment = (JSON.parse(nodePath).split(",").reverse())[0]; + let lastNodeImg = false + if(lastSegment == 'IMG'){ + lastNodeImg = true + } + if(lcpImgUrl != null && lastNodeImg){ + const listJson = JSON.parse(responsivelist); + if(listJson.length > 0){ + for(let i=0;i= 0.75) / COUNTIF(fast_lcp IS NOT NULL) AS pct_good_lcp, + COUNTIF(small_cls / (small_cls + medium_cls + large_cls) >= 0.75) / COUNTIF(small_cls IS NOT NULL) AS pct_good_cls +FROM + `chrome-ux-report.materialized.device_summary` +WHERE + date BETWEEN '2020-01-01' AND '2022-06-01' AND + device IN ('desktop', 'phone') +GROUP BY + date, + device +ORDER BY + date, + device diff --git a/sql/2023/prelcp_domain_sharding.sql b/sql/2023/prelcp_domain_sharding.sql new file mode 100644 index 00000000000..7181d7785cb --- /dev/null +++ b/sql/2023/prelcp_domain_sharding.sql @@ -0,0 +1,58 @@ +WITH requests AS ( + SELECT + _TABLE_SUFFIX AS client, + page, + NET.HOST(url) AS host, + CAST(JSON_QUERY(payload, '$._all_start') AS INT64) AS req_start + FROM + `httparchive.requests.2022_06_01_*` +), + +lcp AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + httparchive.core_web_vitals.GET_LAB_LCP(payload) AS lcp_time + FROM + `httparchive.pages.2022_06_01_*` +), + +hosts AS ( + SELECT + client, + COUNT(DISTINCT host) AS pre_lcp_hosts, + ANY_VALUE(lcp_time) AS lcp + FROM + lcp + JOIN + requests + USING + (client, page) + WHERE + req_start <= lcp_time + GROUP BY + client, + page +) + +SELECT + percentile, + client, + CASE + WHEN lcp <= 2500 THEN 'good' + WHEN lcp > 4000 THEN 'poor' + ELSE 'needs improvement' + END AS lcp, + COUNT(0) AS pages, + APPROX_QUANTILES(pre_lcp_hosts, 1000)[OFFSET(percentile * 10)] AS pre_lcp_hosts +FROM + hosts, + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + percentile, + client, + lcp +ORDER BY + percentile, + client, + lcp diff --git a/sql/2023/render_blocking_resources.sql b/sql/2023/render_blocking_resources.sql new file mode 100644 index 00000000000..508961af546 --- /dev/null +++ b/sql/2023/render_blocking_resources.sql @@ -0,0 +1,22 @@ +WITH lh AS ( + SELECT + IF(STARTS_WITH(_TABLE_SUFFIX, '2021'), '2021', '2022') AS year, + IF(ENDS_WITH(_TABLE_SUFFIX, 'desktop'), 'desktop', 'mobile') AS client, + CAST(JSON_VALUE(report, '$.audits.render-blocking-resources.score') AS FLOAT64) >= 0.9 AS is_passing + FROM + `httparchive.lighthouse.*` + WHERE + _TABLE_SUFFIX IN ('2021_07_01_mobile', '2022_06_01_mobile', '2022_06_01_desktop') +) + +SELECT + year, + client, + COUNTIF(is_passing) AS is_passing, + COUNT(0) AS total, + COUNTIF(is_passing) / COUNT(0) AS pct_passing +FROM + lh +GROUP BY + year, + client diff --git a/sql/2023/ttfb_by_cms.sql b/sql/2023/ttfb_by_cms.sql new file mode 100644 index 00000000000..d2fc69c7105 --- /dev/null +++ b/sql/2023/ttfb_by_cms.sql @@ -0,0 +1,26 @@ +SELECT + app, + client, + COUNT(DISTINCT origin) AS n, + SUM(IF(ttfb.start < 800, ttfb.density, 0)) / SUM(ttfb.density) AS fast, + SUM(IF(ttfb.start >= 800 AND ttfb.start < 1800, ttfb.density, 0)) / SUM(ttfb.density) AS avg, + SUM(IF(ttfb.start >= 1800, ttfb.density, 0)) / SUM(ttfb.density) AS slow +FROM + `chrome-ux-report.all.202206`, + UNNEST(experimental.time_to_first_byte.histogram.bin) AS ttfb +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + * + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + category = 'CMS') +ON + client = IF(form_factor.name = 'desktop', 'desktop', 'mobile') AND + CONCAT(origin, '/') = url +GROUP BY + app, + client +ORDER BY + n DESC diff --git a/sql/2023/ttfb_by_rendering.sql b/sql/2023/ttfb_by_rendering.sql new file mode 100644 index 00000000000..c0ddd2ca935 --- /dev/null +++ b/sql/2023/ttfb_by_rendering.sql @@ -0,0 +1,20 @@ +CREATE TEMPORARY FUNCTION getCSRRatio(payload STRING) RETURNS FLOAT64 LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var stats = JSON.parse($._wpt_bodies); + return (stats.visible_words.rendered / stats.visible_words.raw) || null; +} catch (e) { + return null; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + IF(getCSRRatio(payload) > 1.5, 'CSR', 'SSR') AS rendering, + COUNT(0) AS pages, + APPROX_QUANTILES(httparchive.core_web_vitals.GET_CRUX_TTFB(payload), 1000 IGNORE NULLS)[OFFSET(500)] AS median_ttfb +FROM + `httparchive.pages.2022_06_01_*` +GROUP BY + client, + rendering diff --git a/sql/2023/viewport_meta_zoom_disable.sql b/sql/2023/viewport_meta_zoom_disable.sql new file mode 100644 index 00000000000..8b289a910c6 --- /dev/null +++ b/sql/2023/viewport_meta_zoom_disable.sql @@ -0,0 +1,6 @@ +SELECT + COUNTIF(JSON_VALUE(report, '$.audits.viewport.score') = '0') AS viewport_failed, + COUNT(0) AS total, + COUNTIF(JSON_VALUE(report, '$.audits.viewport.score') = '0') / COUNT(0) AS pct_failed +FROM + `httparchive.lighthouse.2022_06_01_mobile` diff --git a/sql/2023/web_vitals_by_country.sql b/sql/2023/web_vitals_by_country.sql new file mode 100644 index 00000000000..1a7e55424f2 --- /dev/null +++ b/sql/2023/web_vitals_by_country.sql @@ -0,0 +1,176 @@ +#standardSQL +# Core WebVitals by country + +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + NOT IS_GOOD(good, needs_improvement, poor) AND + NOT IS_POOR(good, needs_improvement, poor) +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH +base AS ( + SELECT + origin, + country_code, + + SUM(fast_fid) / SUM(fast_fid + avg_fid + slow_fid) AS fast_fid, + SUM(avg_fid) / SUM(fast_fid + avg_fid + slow_fid) AS avg_fid, + SUM(slow_fid) / SUM(fast_fid + avg_fid + slow_fid) AS slow_fid, + + SUM(fast_inp) / SUM(fast_inp + avg_inp + slow_inp) AS fast_inp, + SUM(avg_inp) / SUM(fast_inp + avg_inp + slow_inp) AS avg_inp, + SUM(slow_inp) / SUM(fast_inp + avg_inp + slow_inp) AS slow_inp, + + SUM(fast_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS fast_lcp, + SUM(avg_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS avg_lcp, + SUM(slow_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS slow_lcp, + + SUM(small_cls) / SUM(small_cls + medium_cls + large_cls) AS small_cls, + SUM(medium_cls) / SUM(small_cls + medium_cls + large_cls) AS medium_cls, + SUM(large_cls) / SUM(small_cls + medium_cls + large_cls) AS large_cls, + + SUM(fast_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS fast_fcp, + SUM(avg_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS avg_fcp, + SUM(slow_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS slow_fcp, + + SUM(fast_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS fast_ttfb, + SUM(avg_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS avg_ttfb, + SUM(slow_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS slow_ttfb + + FROM + `chrome-ux-report.materialized.country_summary` + WHERE + yyyymm = 202206 + GROUP BY + origin, + country_code +) + +SELECT + `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS country, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + base +GROUP BY + country +ORDER BY + total_origins DESC diff --git a/sql/2023/web_vitals_by_device.sql b/sql/2023/web_vitals_by_device.sql new file mode 100644 index 00000000000..59baec43897 --- /dev/null +++ b/sql/2023/web_vitals_by_device.sql @@ -0,0 +1,175 @@ +#standardSQL +# Core WebVitals by device + +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) < 0.75 AND + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) < 0.25 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH +base AS ( + SELECT + date, + origin, + device, + + fast_fid, + avg_fid, + slow_fid, + + fast_inp, + avg_inp, + slow_inp, + + fast_lcp, + avg_lcp, + slow_lcp, + + small_cls, + medium_cls, + large_cls, + + fast_fcp, + avg_fcp, + slow_fcp, + + fast_ttfb, + avg_ttfb, + slow_ttfb + + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + device IN ('desktop', 'phone') AND + date IN ('2020-08-01', '2021-07-01', '2022-06-01') +) + +SELECT + date, + device, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + base +GROUP BY + date, + device diff --git a/sql/2023/web_vitals_by_eff_connection_type.sql b/sql/2023/web_vitals_by_eff_connection_type.sql new file mode 100644 index 00000000000..0864b01956a --- /dev/null +++ b/sql/2023/web_vitals_by_eff_connection_type.sql @@ -0,0 +1,309 @@ +#standardSQL +# WebVitals by effective connection type + +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) < 0.75 AND + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) < 0.25 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH +base AS ( + SELECT + origin, + effective_connection_type.name AS network, + layout_instability, + largest_contentful_paint, + first_input, + first_contentful_paint, + experimental.time_to_first_byte AS time_to_first_byte, + experimental.interaction_to_next_paint AS interaction_to_next_paint + FROM + `chrome-ux-report.all.202206` +), + +cls AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 0.1, bin.density, 0)) AS small, + SUM(IF(bin.start > 0.1 AND bin.start < 0.25, bin.density, 0)) AS medium, + SUM(IF(bin.start >= 0.25, bin.density, 0)) AS large, + `chrome-ux-report`.experimental.PERCENTILE_NUMERIC(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS bin + GROUP BY + origin, + network +), + +lcp AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 2500, bin.density, 0)) AS fast, + SUM(IF(bin.start >= 2500 AND bin.start < 4000, bin.density, 0)) AS avg, + SUM(IF(bin.start >= 4000, bin.density, 0)) AS slow, + `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(largest_contentful_paint.histogram.bin) AS bin + GROUP BY + origin, + network +), + +fid AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 100, bin.density, 0)) AS fast, + SUM(IF(bin.start >= 100 AND bin.start < 300, bin.density, 0)) AS avg, + SUM(IF(bin.start >= 300, bin.density, 0)) AS slow, + `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(first_input.delay.histogram.bin) AS bin + GROUP BY + origin, + network +), + +inp AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 200, bin.density, 0)) AS fast, + SUM(IF(bin.start >= 200 AND bin.start < 500, bin.density, 0)) AS avg, + SUM(IF(bin.start >= 500, bin.density, 0)) AS slow, + `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(interaction_to_next_paint.histogram.bin) AS bin + GROUP BY + origin, + network +), + +fcp AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 1800, bin.density, 0)) AS fast, + SUM(IF(bin.start >= 1800 AND bin.start < 3000, bin.density, 0)) AS avg, + SUM(IF(bin.start >= 3000, bin.density, 0)) AS slow, + `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(first_contentful_paint.histogram.bin) AS bin + GROUP BY + origin, + network +), + +ttfb AS ( + SELECT + origin, + network, + SUM(IF(bin.start < 500, bin.density, 0)) AS fast, + SUM(IF(bin.start >= 500 AND bin.start < 1500, bin.density, 0)) AS avg, + SUM(IF(bin.start >= 1500, bin.density, 0)) AS slow, + `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 + FROM + base + LEFT JOIN + UNNEST(time_to_first_byte.histogram.bin) AS bin + GROUP BY + origin, + network +), + +granular_metrics AS ( + SELECT + origin, + network, + cls.small AS small_cls, + cls.medium AS medium_cls, + cls.large AS large_cls, + cls.p75 AS p75_cls, + + lcp.fast AS fast_lcp, + lcp.avg AS avg_lcp, + lcp.slow AS slow_lcp, + lcp.p75 AS p75_lcp, + + fid.fast AS fast_fid, + fid.avg AS avg_fid, + fid.slow AS slow_fid, + fid.p75 AS p75_fid, + + inp.fast AS fast_inp, + inp.avg AS avg_inp, + inp.slow AS slow_inp, + inp.p75 AS p75_inp, + + fcp.fast AS fast_fcp, + fcp.avg AS avg_fcp, + fcp.slow AS slow_fcp, + fcp.p75 AS p75_fcp, + + ttfb.fast AS fast_ttfb, + ttfb.avg AS avg_ttfb, + ttfb.slow AS slow_ttfb, + ttfb.p75 AS p75_ttfb + FROM + cls + LEFT JOIN + lcp + USING + (origin, network) + LEFT JOIN + fid + USING + (origin, network) + LEFT JOIN + inp + USING + (origin, network) + LEFT JOIN + fcp + USING + (origin, network) + LEFT JOIN + ttfb + USING + (origin, network) +) + +SELECT + network, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + granular_metrics +GROUP BY + network diff --git a/sql/2023/web_vitals_by_rank.sql b/sql/2023/web_vitals_by_rank.sql new file mode 100644 index 00000000000..81241233ba3 --- /dev/null +++ b/sql/2023/web_vitals_by_rank.sql @@ -0,0 +1,182 @@ +#standardSQL +# Core WebVitals by rank + +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + NOT IS_GOOD(good, needs_improvement, poor) AND + NOT IS_POOR(good, needs_improvement, poor) +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH +base AS ( + SELECT + date, + origin, + rank, + + fast_fid, + avg_fid, + slow_fid, + + fast_inp, + avg_inp, + slow_inp, + + fast_lcp, + avg_lcp, + slow_lcp, + + small_cls, + medium_cls, + large_cls, + + fast_fcp, + avg_fcp, + slow_fcp, + + fast_ttfb, + avg_ttfb, + slow_ttfb + + FROM + `chrome-ux-report.materialized.metrics_summary` + WHERE + date IN ('2022-06-01') +) + +SELECT + date, + CASE + WHEN rank_grouping = 10000000 THEN 'all' + ELSE CAST(rank_grouping AS STRING) + END AS ranking, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + base, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping +WHERE + rank <= rank_grouping +GROUP BY + date, + rank_grouping +ORDER BY + rank_grouping diff --git a/sql/2023/web_vitals_by_rank_and_device.sql b/sql/2023/web_vitals_by_rank_and_device.sql new file mode 100644 index 00000000000..efbdd9fa0fd --- /dev/null +++ b/sql/2023/web_vitals_by_rank_and_device.sql @@ -0,0 +1,193 @@ +#standardSQL +# Core WebVitals by rank and device + +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + NOT IS_GOOD(good, needs_improvement, poor) AND + NOT IS_POOR(good, needs_improvement, poor) +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH +base AS ( + SELECT + date, + origin, + device, + rank, + + fast_fid, + avg_fid, + slow_fid, + + fast_inp, + avg_inp, + slow_inp, + + fast_lcp, + avg_lcp, + slow_lcp, + + small_cls, + medium_cls, + large_cls, + + fast_fcp, + avg_fcp, + slow_fcp, + + fast_ttfb, + avg_ttfb, + slow_ttfb + + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + device IN ('desktop', 'phone') AND + date IN ('2022-06-01') +) + +SELECT + date, + device, + rank_grouping AS ranking, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + # Good CWV with optional INP (hypothetical!) + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_inp_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + base, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping +WHERE + rank <= rank_grouping +GROUP BY + date, + device, + rank_grouping +ORDER BY + rank_grouping diff --git a/sql/2023/web_vitals_by_technology.sql b/sql/2023/web_vitals_by_technology.sql new file mode 100644 index 00000000000..f99e73300ea --- /dev/null +++ b/sql/2023/web_vitals_by_technology.sql @@ -0,0 +1,208 @@ +CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 +); + +CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 +); + +CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + NOT IS_GOOD(good, needs_improvement, poor) AND + NOT IS_POOR(good, needs_improvement, poor) +); + +CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( + good + needs_improvement + poor > 0 +); + +WITH base AS ( + SELECT + date, + origin, + CONCAT(origin, '/') AS page, + CASE + WHEN device = 'phone' THEN 'mobile' + ELSE device + END AS client, + + fast_fid, + avg_fid, + slow_fid, + + fast_inp, + avg_inp, + slow_inp, + + fast_lcp, + avg_lcp, + slow_lcp, + + small_cls, + medium_cls, + large_cls, + + fast_fcp, + avg_fcp, + slow_fcp, + + fast_ttfb, + avg_ttfb, + slow_ttfb + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + device IN ('desktop', 'phone') AND + date IN ('2022-06-01') +), + +tech AS ( + SELECT + _TABLE_SUFFIX AS client, + category, + app AS technology, + url AS page + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + category IN ('CMS', 'Ecommerce', 'JavaScript frameworks') +) + +SELECT + date, + client, + category, + technology, + + COUNT(DISTINCT origin) AS total_origins, + + # Good CWV with optional FID + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + + # Good CWV with optional INP (hypothetical!) + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_inp_good, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, + + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor + +FROM + base +JOIN + tech +USING + (client, page) +GROUP BY + date, + client, + category, + technology +HAVING + total_origins >= 1000 +ORDER BY + total_origins DESC From c24de4676901548428598e4b7d7fb8eec6763823 Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Sun, 22 Oct 2023 20:05:23 -0400 Subject: [PATCH 2/7] 2023ify --- sql/2023/README.md | 50 --- sql/2023/bfcache_unload.sql | 2 +- sql/2023/cls_animations.sql | 9 +- sql/2023/cls_unsized_image_height.sql | 9 +- sql/2023/cls_unsized_images.sql | 9 +- sql/2023/correlation_preload_lcp.sql | 84 ----- sql/2023/fid_long_tasks.sql | 35 -- sql/2023/fid_tbt.sql | 13 - sql/2023/gaming_metric.sql | 25 -- sql/2023/inp_long_tasks.sql | 43 +-- sql/2023/inp_tbt.sql | 16 + sql/2023/js_bytes_rank.sql | 12 +- sql/2023/lcp_bytes_distribution.sql | 24 +- sql/2023/lcp_bytes_histogram.sql | 24 +- sql/2023/lcp_element_data.sql | 88 +++-- sql/2023/lcp_element_data_2.sql | 104 ------ sql/2023/lcp_format.sql | 24 +- sql/2023/lcp_host.sql | 11 +- sql/2023/lcp_host_3p.sql | 11 +- sql/2023/lcp_initiator_type.sql | 17 +- sql/2023/lcp_lazy.sql | 19 +- sql/2023/lcp_lazy_technologies.sql | 76 +++++ sql/2023/lcp_lazy_wordpress.sql | 63 ---- sql/2023/lcp_preload.sql | 31 -- sql/2023/lcp_preload_discoverable.sql | 11 +- ..._delay.sql => lcp_resource_load_delay.sql} | 26 +- sql/2023/lcp_resource_type.sql | 11 +- sql/2023/lcp_responsive_data.sql | 41 ++- sql/2023/lcp_wasted_bytes.sql | 39 ++- sql/2023/monthly_cls_lcp.sql | 16 - sql/2023/prelcp_domain_sharding.sql | 58 ---- sql/2023/render_blocking_resources.sql | 22 -- sql/2023/ttfb_by_cms.sql | 26 -- sql/2023/ttfb_by_rendering.sql | 20 -- sql/2023/viewport_meta_zoom_disable.sql | 12 +- sql/2023/web_vitals_by_country.sql | 176 ---------- sql/2023/web_vitals_by_device.sql | 17 +- .../web_vitals_by_eff_connection_type.sql | 309 ------------------ sql/2023/web_vitals_by_rank.sql | 182 ----------- sql/2023/web_vitals_by_rank_and_device.sql | 13 +- sql/2023/web_vitals_by_technology.sql | 208 ------------ 41 files changed, 358 insertions(+), 1628 deletions(-) delete mode 100644 sql/2023/README.md delete mode 100644 sql/2023/correlation_preload_lcp.sql delete mode 100644 sql/2023/fid_long_tasks.sql delete mode 100644 sql/2023/fid_tbt.sql delete mode 100644 sql/2023/gaming_metric.sql create mode 100644 sql/2023/inp_tbt.sql delete mode 100644 sql/2023/lcp_element_data_2.sql create mode 100644 sql/2023/lcp_lazy_technologies.sql delete mode 100644 sql/2023/lcp_lazy_wordpress.sql delete mode 100644 sql/2023/lcp_preload.sql rename sql/2023/{lcp_resource_delay.sql => lcp_resource_load_delay.sql} (51%) delete mode 100644 sql/2023/monthly_cls_lcp.sql delete mode 100644 sql/2023/prelcp_domain_sharding.sql delete mode 100644 sql/2023/render_blocking_resources.sql delete mode 100644 sql/2023/ttfb_by_cms.sql delete mode 100644 sql/2023/ttfb_by_rendering.sql delete mode 100644 sql/2023/web_vitals_by_country.sql delete mode 100644 sql/2023/web_vitals_by_eff_connection_type.sql delete mode 100644 sql/2023/web_vitals_by_rank.sql delete mode 100644 sql/2023/web_vitals_by_technology.sql diff --git a/sql/2023/README.md b/sql/2023/README.md deleted file mode 100644 index fea835c8619..00000000000 --- a/sql/2023/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# 2022 Performance queries - - - -## Resources - -- [📄 Planning doc][~google-doc] -- [📊 Results sheet][~google-sheets] -- [📝 Markdown file][~chapter-markdown] - -[~google-doc]: https://docs.google.com/document/d/1IKV40fllCZTqeu-R6-73ckjQR9S6jiBfVBBfdcpAMkI/edit?usp=sharing -[~google-sheets]: https://docs.google.com/spreadsheets/d/1TPA_4xRTBB2fQZaBPZHVFvD0ikrR-4sNkfJfUEpjibs/edit?usp=sharing -[~chapter-markdown]: https://github.com/HTTPArchive/almanac.httparchive.org/tree/main/src/content/en/2022/performance.md - -## Query List - -Taken from [`Metrics` Section](https://docs.google.com/document/d/1IKV40fllCZTqeu-R6-73ckjQR9S6jiBfVBBfdcpAMkI/edit#heading=h.zbvh8yhwkp2i) in planning doc, where technical notes on how queries might be formulated live. - -## Query notes - -### `inp_long_tasks.sql` - -This query generates a scatterplot of field-based, page-level p75 INP versus the sum of all lab-based long tasks on the page according to Lighthouse. - -Lab-based mobile CPU throttling necessarily means that long tasks should be higher than desktop. It's interesting to look at the trendlines for both desktop and mobile to see that higher INP correlates with higher long tasks. - -### `lcp_resource_delay.sql` - -This query subtracts the lab-based TTFB time from the lab-based LCP element's request start time and generates a distribution over all pages. - -The `lcp_elem_stats.startTime` and `lcp_resource.timestamp` values did not seem to correspond to the actual LCP element request start times seen in the WPT results, so the query goes the more expensive route to join the pages data with the more reliable requests data. - -### `prelcp_domain_sharding.sql` - -This query takes the distribution of the number of unique hosts connected prior to the lab-based LCP time, broken down by whether the LCP was good, NI, or poor. - -### `ttfb_by_rendering.sql` - -This query segments pages by whether they use client-side rendering (CSR) or server-side rendering (SSR). There is no high quality signal for CSR/SSR so the heuristic used in this query is the ratio of the number of words in the document served in the static HTML to the number of words in the final rendered page. If the number of words increases by 1.5x after rendering, then we consider the page to use CSR. - -Using both rendering buckets, we then calculate the median field-based p75 TTFB. - -This metric is not expected to be very meaningful given the tenuous definition of CSR. diff --git a/sql/2023/bfcache_unload.sql b/sql/2023/bfcache_unload.sql index 8aecd755ba1..8e194a5b9fb 100644 --- a/sql/2023/bfcache_unload.sql +++ b/sql/2023/bfcache_unload.sql @@ -20,7 +20,7 @@ SELECT COUNTIF(has_unload) / COUNT(0) AS pct FROM lh, - UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS _rank + UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank WHERE rank <= _rank GROUP BY diff --git a/sql/2023/cls_animations.sql b/sql/2023/cls_animations.sql index 1452e404fc4..61aaac232eb 100644 --- a/sql/2023/cls_animations.sql +++ b/sql/2023/cls_animations.sql @@ -1,9 +1,12 @@ WITH lh AS ( SELECT - _TABLE_SUFFIX AS client, - ARRAY_LENGTH(JSON_QUERY_ARRAY(report, '$.audits.non-composited-animations.details.items')) AS num_animations + client, + ARRAY_LENGTH(JSON_QUERY_ARRAY(lighthouse, '$.audits.non-composited-animations.details.items')) AS num_animations FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/cls_unsized_image_height.sql b/sql/2023/cls_unsized_image_height.sql index 2b853dfa73a..4c3ad5cef65 100644 --- a/sql/2023/cls_unsized_image_height.sql +++ b/sql/2023/cls_unsized_image_height.sql @@ -1,10 +1,13 @@ WITH lh AS ( SELECT - _TABLE_SUFFIX AS client, + client, CAST(JSON_VALUE(unsized_image, '$.node.boundingRect.height') AS INT64) AS height FROM - `httparchive.lighthouse.2022_06_01_*`, - UNNEST(JSON_QUERY_ARRAY(report, '$.audits.unsized-images.details.items')) AS unsized_image + `httparchive.all.pages`, + UNNEST(JSON_QUERY_ARRAY(lighthouse, '$.audits.unsized-images.details.items')) AS unsized_image + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/cls_unsized_images.sql b/sql/2023/cls_unsized_images.sql index 5a5bd4c2e39..cba6c373a67 100644 --- a/sql/2023/cls_unsized_images.sql +++ b/sql/2023/cls_unsized_images.sql @@ -1,9 +1,12 @@ WITH lh AS ( SELECT - _TABLE_SUFFIX AS client, - ARRAY_LENGTH(JSON_QUERY_ARRAY(report, '$.audits.unsized-images.details.items')) AS num_unsized_images + client, + ARRAY_LENGTH(JSON_QUERY_ARRAY(lighthouse, '$.audits.unsized-images.details.items')) AS num_unsized_images FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/correlation_preload_lcp.sql b/sql/2023/correlation_preload_lcp.sql deleted file mode 100644 index ce5329028b0..00000000000 --- a/sql/2023/correlation_preload_lcp.sql +++ /dev/null @@ -1,84 +0,0 @@ -CREATE TEMPORARY FUNCTION getResourceHints(payload STRING) -RETURNS STRUCT -LANGUAGE js AS ''' -var hints = ['preload']; -try { - var $ = JSON.parse(payload); - var almanac = JSON.parse($._almanac); - return hints.reduce((results, hint) => { - // Null values are omitted from BigQuery aggregations. - // This means only pages with at least one hint are considered. - results[hint] = almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).length || 0; - return results; - }, {}); -} catch (e) { - return hints.reduce((results, hint) => { - results[hint] = 0; - return results; - }, {}); -} -'''; - -CREATE TEMPORARY FUNCTION getGoodCwv(payload STRING) -RETURNS STRUCT -LANGUAGE js AS ''' -try { - var $ = JSON.parse(payload); - var crux = $._CrUX; - - if (crux) { - return Object.keys(crux.metrics).reduce((acc, n) => ({ - ...acc, - [n]: crux.metrics[n].histogram[0].density > 0.75 - }), {}) - } - - return null; -} catch (e) { - return null; -} -'''; - -SELECT - device, - - LEAST(hints.preload, 30) AS preload, - - COUNT(0) AS freq, - SUM(COUNT(0)) OVER (PARTITION BY device) AS total, - - COUNTIF(CrUX.largest_contentful_paint) AS lcp_good, - COUNTIF(CrUX.largest_contentful_paint IS NOT NULL) AS any_lcp, - COUNTIF(CrUX.largest_contentful_paint) / COUNTIF(CrUX.largest_contentful_paint IS NOT NULL) AS pct_lcp_good, - - COUNTIF(CrUX.first_input_delay) AS fid_good, - COUNTIF(CrUX.first_input_delay IS NOT NULL) AS any_fid, - COUNTIF(CrUX.first_input_delay) / COUNTIF(CrUX.first_input_delay IS NOT NULL) AS pct_fid_good, - - COUNTIF(CrUX.cumulative_layout_shift) AS cls_good, - COUNTIF(CrUX.cumulative_layout_shift IS NOT NULL) AS any_cls, - COUNTIF(CrUX.cumulative_layout_shift) / COUNTIF(CrUX.cumulative_layout_shift IS NOT NULL) AS pct_cls_good, - - COUNTIF(CrUX.first_contentful_paint) AS fcp_good, - COUNTIF(CrUX.first_contentful_paint IS NOT NULL) AS any_fcp, - COUNTIF(CrUX.first_contentful_paint) / COUNTIF(CrUX.first_contentful_paint IS NOT NULL) AS pct_fcp_good, - - COUNTIF(CrUX.largest_contentful_paint AND CrUX.first_input_delay IS NOT FALSE AND CrUX.cumulative_layout_shift) AS cwv_good, - COUNTIF(CrUX.largest_contentful_paint IS NOT NULL AND CrUX.cumulative_layout_shift IS NOT NULL) AS eligible_cwv, - COUNTIF(CrUX.largest_contentful_paint AND CrUX.first_input_delay IS NOT FALSE AND CrUX.cumulative_layout_shift) / COUNTIF(CrUX.largest_contentful_paint IS NOT NULL AND CrUX.cumulative_layout_shift IS NOT NULL) AS pct_cwv_good -FROM ( - SELECT - _TABLE_SUFFIX AS device, - getResourceHints(payload) AS hints, - getGoodCwv(payload) AS CrUX - FROM - `httparchive.pages.2022_06_01_*` -) -WHERE - CrUX IS NOT NULL -GROUP BY - device, - preload -ORDER BY - device, - preload diff --git a/sql/2023/fid_long_tasks.sql b/sql/2023/fid_long_tasks.sql deleted file mode 100644 index 6fec7d7bb5b..00000000000 --- a/sql/2023/fid_long_tasks.sql +++ /dev/null @@ -1,35 +0,0 @@ -WITH long_tasks AS ( - SELECT - _TABLE_SUFFIX AS client, - url AS page, - CAST(JSON_QUERY(item, '$.duration') AS FLOAT64) AS long_task_duration - FROM - `lighthouse.2022_06_01_*`, - UNNEST(JSON_QUERY_ARRAY(report, '$.audits.long-tasks.details.items')) AS item -), - -per_page AS ( - SELECT - client, - page, - SUM(long_task_duration) AS long_tasks - FROM - long_tasks - GROUP BY - client, - page -) - -SELECT - percentile, - client, - APPROX_QUANTILES(long_tasks, 1000)[OFFSET(percentile * 10)] AS long_tasks -FROM - per_page, - UNNEST([10, 25, 50, 75, 90, 100]) AS percentile -GROUP BY - percentile, - client -ORDER BY - percentile, - client diff --git a/sql/2023/fid_tbt.sql b/sql/2023/fid_tbt.sql deleted file mode 100644 index a4bf92f8dff..00000000000 --- a/sql/2023/fid_tbt.sql +++ /dev/null @@ -1,13 +0,0 @@ -SELECT - percentile, - _TABLE_SUFFIX AS client, - APPROX_QUANTILES(CAST(JSON_QUERY(report, '$.audits.total-blocking-time.numericValue') AS FLOAT64), 1000)[OFFSET(percentile * 10)] AS tbt -FROM - `httparchive.lighthouse.2022_06_01_*`, - UNNEST([10, 25, 50, 75, 90, 100]) AS percentile -GROUP BY - percentile, - client -ORDER BY - percentile, - client diff --git a/sql/2023/gaming_metric.sql b/sql/2023/gaming_metric.sql deleted file mode 100644 index 567430c4145..00000000000 --- a/sql/2023/gaming_metric.sql +++ /dev/null @@ -1,25 +0,0 @@ -SELECT - _TABLE_SUFFIX AS client, - COUNT(0) AS total, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-ChromeLH') = 'true', 1, 0)) AS chrome_lighthouse, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-ChromeLH') = 'true', 1, 0)) / COUNT(0) AS chrome_lighthouse_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-GTmetrix') = 'true', 1, 0)) AS gtmetrix, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-GTmetrix') = 'true', 1, 0)) / COUNT(0) AS gtmetrix_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-PageSpeed') = 'true', 1, 0)) AS pagespeed, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.detectUA-PageSpeed') = 'true', 1, 0)) / COUNT(0) AS pagespeed_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationStrict') = 'true', 1, 0)) AS img_animation_strict, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationStrict') = 'true', 1, 0)) / COUNT(0) AS img_animation_strict_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationSoft') = 'true', 1, 0)) AS img_animation_soft, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.imgAnimationSoft') = 'true', 1, 0)) / COUNT(0) AS img_animation_soft_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlayStrict') = 'true', 1, 0)) AS fid_iframe_overlay_strict, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlayStrict') = 'true', 1, 0)) / COUNT(0) AS fid_iframe_overlay_strict_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlaySoft') = 'true', 1, 0)) AS fid_iframe_overlay_soft, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.fidIframeOverlaySoft') = 'true', 1, 0)) / COUNT(0) AS fid_iframe_overlay_soft_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlayStrict') = 'true', 1, 0)) AS lcp_overlay_strict, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlayStrict') = 'true', 1, 0)) / COUNT(0) AS lcp_overlay_strict_per, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlaySoft') = 'true', 1, 0)) AS lcp_overlay_soft, - SUM(IF(JSON_EXTRACT_SCALAR(payload, '$._performance.gaming_metrics.lcpOverlaySoft') = 'true', 1, 0)) / COUNT(0) AS lcp_overlay_soft_per -FROM - `httparchive.pages.2022_06_01_*` -GROUP BY - client diff --git a/sql/2023/inp_long_tasks.sql b/sql/2023/inp_long_tasks.sql index 22f9f4cb98e..61f5697dac3 100644 --- a/sql/2023/inp_long_tasks.sql +++ b/sql/2023/inp_long_tasks.sql @@ -1,54 +1,27 @@ WITH long_tasks AS ( - SELECT - _TABLE_SUFFIX AS client, - url AS page, - CAST(JSON_QUERY(item, '$.duration') AS FLOAT64) AS long_task_duration - FROM - `lighthouse.2022_06_01_*`, - UNNEST(JSON_QUERY_ARRAY(report, '$.audits.long-tasks.details.items')) AS item -), - -per_page AS ( SELECT client, page, - SUM(long_task_duration) AS long_tasks + ANY_VALUE(httparchive.core_web_vitals.GET_CRUX_INP(payload)) AS inp, + SUM(CAST(JSON_QUERY(item, '$.duration') AS FLOAT64)) AS long_tasks FROM - long_tasks + `httparchive.all.pages`, + UNNEST(JSON_QUERY_ARRAY(lighthouse, '$.audits.long-tasks.details.items')) AS item + WHERE + date = '2023-10-01' AND + is_root_page GROUP BY client, page ), -crux_inp AS ( - SELECT - _TABLE_SUFFIX AS client, - url AS page, - httparchive.core_web_vitals.GET_CRUX_INP(payload) AS inp - FROM - `httparchive.pages.2022_06_01_*` -), - -combined AS ( - SELECT - client, - long_tasks, - inp - FROM - per_page - JOIN - crux_inp - USING - (client, page) -), - meta AS ( SELECT *, COUNT(0) OVER (PARTITION BY client) AS n, ROW_NUMBER() OVER (PARTITION BY client ORDER BY inp) AS row FROM - combined + long_tasks WHERE inp IS NOT NULL ) diff --git a/sql/2023/inp_tbt.sql b/sql/2023/inp_tbt.sql new file mode 100644 index 00000000000..5175b3a1b80 --- /dev/null +++ b/sql/2023/inp_tbt.sql @@ -0,0 +1,16 @@ +SELECT + percentile, + client, + APPROX_QUANTILES(CAST(JSON_QUERY(lighthouse, '$.audits.total-blocking-time.numericValue') AS FLOAT64), 1000)[OFFSET(percentile * 10)] AS tbt +FROM + `httparchive.all.pages`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +WHERE + date = '2023-10-01' AND + is_root_page +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2023/js_bytes_rank.sql b/sql/2023/js_bytes_rank.sql index ae0f3d25e82..dbb1cefa266 100644 --- a/sql/2023/js_bytes_rank.sql +++ b/sql/2023/js_bytes_rank.sql @@ -1,11 +1,13 @@ SELECT - IF(_rank < 10000000, CAST(_rank AS STRING), 'all') AS rank, - _TABLE_SUFFIX AS client, - APPROX_QUANTILES(bytesJS, 1001)[OFFSET(501)] / 1024 AS js_kbytes + IF(_rank < 100000000, CAST(_rank AS STRING), 'all') AS rank, + client, + APPROX_QUANTILES(CAST(JSON_VALUE(summary, '$.bytesJS') AS INT64), 1000)[OFFSET(500)] / 1024 AS js_kbytes FROM - `httparchive.summary_pages.2022_06_01_*`, - UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS _rank + `httparchive.all.pages`, + UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank WHERE + date = '2023-10-01' AND + is_root_page AND rank <= _rank GROUP BY rank, diff --git a/sql/2023/lcp_bytes_distribution.sql b/sql/2023/lcp_bytes_distribution.sql index 7ed7e0dcd70..43ccd956b46 100644 --- a/sql/2023/lcp_bytes_distribution.sql +++ b/sql/2023/lcp_bytes_distribution.sql @@ -1,20 +1,26 @@ WITH pages AS ( SELECT - _TABLE_SUFFIX AS client, - CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ), requests AS ( SELECT - _TABLE_SUFFIX AS client, - pageid, + client, + page, url, - respSize / 1024 AS kbytes + CAST(JSON_VALUE(summary, '$.respSize') AS INT64) / 1024 AS kbytes FROM - `httparchive.summary_requests.2022_06_01_*` + `httparchive.all.requests` + WHERE + date = '2023-10-01' AND + is_root_page ) SELECT @@ -26,7 +32,7 @@ FROM JOIN requests USING - (client, pageid, url), + (client, page, url), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile GROUP BY percentile, diff --git a/sql/2023/lcp_bytes_histogram.sql b/sql/2023/lcp_bytes_histogram.sql index 49c3ec54e58..a03fff1b6e4 100644 --- a/sql/2023/lcp_bytes_histogram.sql +++ b/sql/2023/lcp_bytes_histogram.sql @@ -1,20 +1,26 @@ WITH pages AS ( SELECT - _TABLE_SUFFIX AS client, - CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ), requests AS ( SELECT - _TABLE_SUFFIX AS client, - pageid, + client, + page, url, - respSize / 1024 AS kbytes + CAST(JSON_VALUE(summary, '$.respSize') AS INT64) / 1024 AS kbytes FROM - `httparchive.summary_requests.2022_06_01_*` + `httparchive.all.requests` + WHERE + date = '2023-10-01' AND + is_root_page ) SELECT @@ -28,7 +34,7 @@ FROM JOIN requests USING - (client, pageid, url) + (client, page, url) GROUP BY client, kbytes diff --git a/sql/2023/lcp_element_data.sql b/sql/2023/lcp_element_data.sql index a3d8460c60e..a2ca420dc23 100644 --- a/sql/2023/lcp_element_data.sql +++ b/sql/2023/lcp_element_data.sql @@ -1,6 +1,3 @@ -#standardSQL -# LCP element node details - CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' try { const data = JSON.parse(attributes); @@ -21,6 +18,16 @@ CREATE TEMP FUNCTION getDecodingAttr(attributes STRING) RETURNS STRING LANGUAGE } '''; +CREATE TEMP FUNCTION getFetchPriorityAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const fetchPriorityAttr = data.find(attr => attr["name"] === "fetchpriority") + return fetchPriorityAttr.value + } catch (e) { + return ""; + } +'''; + CREATE TEMP FUNCTION getLoadingClasses(attributes STRING) RETURNS STRING LANGUAGE js AS ''' try { const data = JSON.parse(attributes); @@ -41,7 +48,7 @@ LANGUAGE js AS ''' var hints = ['preload', 'prefetch', 'preconnect', 'prerender', 'dns-prefetch', 'modulepreload']; try { var $ = JSON.parse(payload); - var almanac = JSON.parse($._almanac); + var almanac = JSON.parse($.almanac); return hints.reduce((results, hint) => { results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint); return results; @@ -54,74 +61,63 @@ try { } '''; -CREATE TEMPORARY FUNCTION getFetchPriority(payload STRING) -RETURNS STRUCT -LANGUAGE js AS ''' -var hints = ['high', 'low', 'auto']; -try { - var $ = JSON.parse(payload); - var almanac = JSON.parse($._almanac); - return hints.reduce((results, hint) => { - results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint); - return results; - }, {}); -} catch (e) { - return hints.reduce((results, hint) => { - results[hint] = false; - return results; - }, {}); -} -'''; -WITH -lcp_stats AS ( +WITH lcp_stats AS ( SELECT - _TABLE_SUFFIX AS client, - url, - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') AS nodeName, - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.url') AS elementUrl, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.size') AS INT64) AS size, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.loadTime') AS FLOAT64) AS loadTime, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.startTime') AS FLOAT64) AS startTime, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.renderTime') AS FLOAT64) AS renderTime, - JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes') AS attributes, - getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS loading, - getDecodingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS decoding, - getLoadingClasses(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS classWithLazyload, - getResourceHints(payload) AS hints + client, + page, + JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.nodeName') AS nodeName, + JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.url') AS elementUrl, + CAST(JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.size') AS INT64) AS size, + CAST(JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.loadTime') AS FLOAT64) AS loadTime, + CAST(JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.startTime') AS FLOAT64) AS startTime, + CAST(JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.renderTime') AS FLOAT64) AS renderTime, + JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes') AS attributes, + getLoadingAttr(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS loading, + getDecodingAttr(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS decoding, + getLoadingClasses(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS classWithLazyload, + getFetchPriorityAttr(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS fetchPriority, + getResourceHints(custom_metrics) AS hints FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) SELECT client, nodeName, - COUNT(DISTINCT url) AS pages, + COUNT(DISTINCT page) AS pages, ANY_VALUE(total) AS total, - COUNT(DISTINCT url) / ANY_VALUE(total) AS pct, + COUNT(DISTINCT page) / ANY_VALUE(total) AS pct, COUNTIF(elementUrl != '') AS haveImages, - COUNTIF(elementUrl != '') / COUNT(DISTINCT url) AS pct_haveImages, + COUNTIF(elementUrl != '') / COUNT(DISTINCT page) AS pct_haveImages, COUNTIF(loading = 'eager') AS native_eagerload, COUNTIF(loading = 'lazy') AS native_lazyload, COUNTIF(classWithLazyload != '') AS lazyload_class, COUNTIF(classWithLazyload != '' OR loading = 'lazy') AS probably_lazyLoaded, - COUNTIF(classWithLazyload != '' OR loading = 'lazy') / COUNT(DISTINCT url) AS pct_prob_lazyloaded, + COUNTIF(classWithLazyload != '' OR loading = 'lazy') / COUNT(DISTINCT page) AS pct_prob_lazyloaded, COUNTIF(decoding = 'async') AS async_decoding, COUNTIF(decoding = 'sync') AS sync_decoding, COUNTIF(decoding = 'auto') AS auto_decoding, - COUNT(0) AS total, + COUNTIF(fetchPriority = 'low') AS priority_low, + COUNTIF(fetchPriority = 'high') AS priority_high, COUNTIF(hints.preload) AS preload, COUNTIF(hints.preload) / COUNT(0) AS pct_preload FROM lcp_stats JOIN ( SELECT - _TABLE_SUFFIX AS client, + client, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page GROUP BY - _TABLE_SUFFIX) + client) USING (client) GROUP BY diff --git a/sql/2023/lcp_element_data_2.sql b/sql/2023/lcp_element_data_2.sql deleted file mode 100644 index d5d2e4207ec..00000000000 --- a/sql/2023/lcp_element_data_2.sql +++ /dev/null @@ -1,104 +0,0 @@ -#standardSQL -# LCP element node details - -CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const loadingAttr = data.find(attr => attr["name"] === "loading") - return loadingAttr.value - } catch (e) { - return ""; - } -'''; - -CREATE TEMP FUNCTION getDecodingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const decodingAttr = data.find(attr => attr["name"] === "decoding") - return decodingAttr.value - } catch (e) { - return ""; - } -'''; - -CREATE TEMP FUNCTION getFetchPriorityAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const fetchPriorityAttr = data.find(attr => attr["name"] === "fetchpriority") - return fetchPriorityAttr.value - } catch (e) { - return ""; - } -'''; - -CREATE TEMP FUNCTION getLoadingClasses(attributes STRING) RETURNS STRING LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const classes = data.find(attr => attr["name"] === "class").value - if (classes.indexOf('lazyload') !== -1) { - return classes - } else { - return "" - } - } catch (e) { - return ""; - } -'''; - -WITH -lcp_stats AS ( - SELECT - _TABLE_SUFFIX AS client, - url, - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') AS nodeName, - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.url') AS elementUrl, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.size') AS INT64) AS size, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.loadTime') AS FLOAT64) AS loadTime, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.startTime') AS FLOAT64) AS startTime, - CAST(JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.renderTime') AS FLOAT64) AS renderTime, - JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes') AS attributes, - getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS loading, - getDecodingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS decoding, - getLoadingClasses(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS classWithLazyload, - getFetchPriorityAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS fetchPriority - FROM - `httparchive.pages.2022_06_01_*` -) - -SELECT - client, - nodeName, - COUNT(DISTINCT url) AS pages, - ANY_VALUE(total) AS total, - COUNT(DISTINCT url) / ANY_VALUE(total) AS pct, - COUNTIF(elementUrl != '') AS haveImages, - COUNTIF(elementUrl != '') / COUNT(DISTINCT url) AS pct_haveImages, - COUNTIF(loading = 'eager') AS native_eagerload, - COUNTIF(loading = 'lazy') AS native_lazyload, - COUNTIF(classWithLazyload != '') AS lazyload_class, - COUNTIF(classWithLazyload != '' OR loading = 'lazy') AS probably_lazyLoaded, - COUNTIF(classWithLazyload != '' OR loading = 'lazy') / COUNT(DISTINCT url) AS pct_prob_lazyloaded, - COUNTIF(decoding = 'async') AS async_decoding, - COUNTIF(decoding = 'sync') AS sync_decoding, - COUNTIF(decoding = 'auto') AS auto_decoding, - COUNTIF(fetchPriority = 'low') AS priority_low, - COUNTIF(fetchPriority = 'high') AS priority_high -FROM - lcp_stats -JOIN ( - SELECT - _TABLE_SUFFIX AS client, - COUNT(0) AS total - FROM - `httparchive.pages.2022_06_01_*` - GROUP BY - _TABLE_SUFFIX) -USING - (client) -GROUP BY - client, - nodeName -HAVING - pages > 1000 -ORDER BY - pct DESC diff --git a/sql/2023/lcp_format.sql b/sql/2023/lcp_format.sql index 7bf15f18d6c..61aba286ce1 100644 --- a/sql/2023/lcp_format.sql +++ b/sql/2023/lcp_format.sql @@ -1,20 +1,26 @@ WITH pages AS ( SELECT - _TABLE_SUFFIX AS client, - CAST(JSON_VALUE(payload, '$._metadata.page_id') AS INT64) AS pageid, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ), requests AS ( SELECT - _TABLE_SUFFIX AS client, - pageid, + client, + page, url, - format + JSON_VALUE(summary, '$.format') AS format FROM - `httparchive.summary_requests.2022_06_01_*` + `httparchive.all.requests` + WHERE + date = '2023-10-01' AND + is_root_page ) SELECT @@ -28,7 +34,7 @@ FROM JOIN requests USING - (client, pageid, url) + (client, page, url) GROUP BY client, format diff --git a/sql/2023/lcp_host.sql b/sql/2023/lcp_host.sql index 528fec937c8..6f051899a56 100644 --- a/sql/2023/lcp_host.sql +++ b/sql/2023/lcp_host.sql @@ -1,10 +1,13 @@ WITH lcp AS ( SELECT - _TABLE_SUFFIX AS client, - url AS page, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/lcp_host_3p.sql b/sql/2023/lcp_host_3p.sql index 4fda76c04e9..ae84efc1196 100644 --- a/sql/2023/lcp_host_3p.sql +++ b/sql/2023/lcp_host_3p.sql @@ -1,10 +1,13 @@ WITH lcp AS ( SELECT - _TABLE_SUFFIX AS client, - url AS page, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/lcp_initiator_type.sql b/sql/2023/lcp_initiator_type.sql index dbb611ffb3b..73040c08e35 100644 --- a/sql/2023/lcp_initiator_type.sql +++ b/sql/2023/lcp_initiator_type.sql @@ -1,11 +1,14 @@ WITH lcp AS ( SELECT - _TABLE_SUFFIX AS client, - url AS page, - JSON_VALUE(payload, '$._performance.lcp_resource.initiator.url') AS url, - JSON_VALUE(payload, '$._performance.is_lcp_statically_discoverable') = 'false' AS not_discoverable + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url, + JSON_VALUE(custom_metrics, '$.performance.is_lcp_statically_discoverable') = 'false' AS not_discoverable FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ), requests AS ( @@ -15,9 +18,9 @@ requests AS ( url, type FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' + date = '2023-10-01' ) diff --git a/sql/2023/lcp_lazy.sql b/sql/2023/lcp_lazy.sql index 23fd7540827..d41948742ef 100644 --- a/sql/2023/lcp_lazy.sql +++ b/sql/2023/lcp_lazy.sql @@ -1,10 +1,10 @@ -CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' +CREATE TEMP FUNCTION isLazyLoaded(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' try { const data = JSON.parse(attributes); const loadingAttr = data.find(attr => attr["name"] === "loading") - return loadingAttr.value + return loadingAttr.value == 'lazy' } catch (e) { - return ""; + return null; } '''; @@ -23,14 +23,15 @@ CREATE TEMP FUNCTION hasLazyHeuristics(attributes STRING) RETURNS BOOLEAN LANGUA WITH lcp_stats AS ( SELECT - _TABLE_SUFFIX AS client, - getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) = 'lazy' AS native_lazy, - hasLazyHeuristics(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS custom_lazy + client, + isLazyLoaded(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS native_lazy, + hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy, FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` WHERE - # LCP only. - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') = 'IMG' + date = '2023-10-01' AND + is_root_page AND + JSON_EXTRACT_SCALAR(custom_metrics, '$.performance.lcp_elem_stats.nodeName') = 'IMG' ) SELECT diff --git a/sql/2023/lcp_lazy_technologies.sql b/sql/2023/lcp_lazy_technologies.sql new file mode 100644 index 00000000000..ab3f49c3d87 --- /dev/null +++ b/sql/2023/lcp_lazy_technologies.sql @@ -0,0 +1,76 @@ +CREATE TEMP FUNCTION isLazyLoaded(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const loadingAttr = data.find(attr => attr["name"] === "loading") + return loadingAttr.value == 'lazy' + } catch (e) { + return null; + } +'''; + +CREATE TEMP FUNCTION hasLazyHeuristics(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' + try { + const data = JSON.parse(attributes); + const classes = data.find(attr => attr["name"] === "class").value; + const hasLazyClasses = classes.indexOf('lazyload') !== -1; + const hasLazySrc = data.includes(attr => attr["name"] === "data-src"); + + return hasLazyClasses || hasLazySrc; + } catch (e) { + return false; + } +'''; + +WITH lazy_tech AS ( + SELECT + client, + page, + isLazyLoaded(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS native_lazy, + hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy, + t.technology + FROM + `httparchive.all.pages`, + UNNEST(technologies) AS t + WHERE + date = '2023-10-01' AND + is_root_page +), + +tech_totals AS ( + SELECT + client, + technology, + COUNT(0) AS pages_per_technology + FROM + lazy_tech + GROUP BY + client, + technology +) + + +SELECT + client, + technology, + COUNTIF(native_lazy) AS native_lazy, + COUNTIF(custom_lazy) AS custom_lazy, + COUNTIF(native_lazy OR custom_lazy) AS either_lazy, + COUNT(0) AS pages, + COUNTIF(native_lazy) / COUNT(0) AS pct_native_lazy, + COUNTIF(custom_lazy) / COUNT(0) AS pct_custom_lazy, + COUNTIF(native_lazy OR custom_lazy) / COUNT(0) AS pct_either_lazy +FROM + lazy_tech +JOIN + tech_totals +USING + (client, technology) +GROUP BY + client, + technology +HAVING + pages > 1000 AND + pct_either_lazy > 0.1 +ORDER BY + either_lazy DESC + diff --git a/sql/2023/lcp_lazy_wordpress.sql b/sql/2023/lcp_lazy_wordpress.sql deleted file mode 100644 index 9194ec592af..00000000000 --- a/sql/2023/lcp_lazy_wordpress.sql +++ /dev/null @@ -1,63 +0,0 @@ -CREATE TEMP FUNCTION getLoadingAttr(attributes STRING) RETURNS STRING LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const loadingAttr = data.find(attr => attr["name"] === "loading") - return loadingAttr.value - } catch (e) { - return ""; - } -'''; - -CREATE TEMP FUNCTION hasLazyHeuristics(attributes STRING) RETURNS BOOLEAN LANGUAGE js AS ''' - try { - const data = JSON.parse(attributes); - const classes = data.find(attr => attr["name"] === "class").value; - const hasLazyClasses = classes.indexOf('lazyload') !== -1; - const hasLazySrc = data.includes(attr => attr["name"] === "data-src"); - - return hasLazyClasses || hasLazySrc; - } catch (e) { - return false; - } -'''; - -WITH lazy AS ( - SELECT - _TABLE_SUFFIX AS client, - url, - getLoadingAttr(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) = 'lazy' AS native_lazy, - hasLazyHeuristics(JSON_EXTRACT(payload, '$._performance.lcp_elem_stats.attributes')) AS custom_lazy - FROM - `httparchive.pages.2022_06_01_*` - WHERE - # LCP only. - JSON_EXTRACT_SCALAR(payload, '$._performance.lcp_elem_stats.nodeName') = 'IMG' -), - -wp AS ( - SELECT DISTINCT - _TABLE_SUFFIX AS client, - url, - TRUE AS wordpress - FROM - `httparchive.technologies.2022_06_01_*` - WHERE - app = 'WordPress' -) - - -SELECT - client, - COUNTIF(wordpress) AS wordpress, - COUNTIF(native_lazy) AS native_lazy, - COUNTIF(custom_lazy) AS custom_lazy, - COUNTIF(wordpress AND native_lazy) / COUNTIF(native_lazy) AS pct_wordpress_native_lazy, - COUNTIF(wordpress AND custom_lazy) / COUNTIF(custom_lazy) AS pct_wordpress_custom_lazy -FROM - lazy -LEFT JOIN - wp -USING - (client, url) -GROUP BY - client diff --git a/sql/2023/lcp_preload.sql b/sql/2023/lcp_preload.sql deleted file mode 100644 index b814e52c4b9..00000000000 --- a/sql/2023/lcp_preload.sql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE TEMPORARY FUNCTION isLCPPreloaded(payload STRING) RETURNS BOOLEAN LANGUAGE js AS ''' -try { - var $ = JSON.parse(payload); - var lcp_url = $._performance.lcp_elem_stats.url; - - var almanac = JSON.parse($._almanac); - return !!almanac['link-nodes'].nodes.find(link => { - return link.rel && link.rel.toLowerCase() == 'preload' && lcp_url.endsWith(link.href); - }); -} catch (e) { - return false; -} -'''; - -WITH preloaded AS ( - SELECT - _TABLE_SUFFIX AS client, - isLCPPreloaded(payload) AS is_lcp_preloaded - FROM - `httparchive.pages.2022_06_01_*` -) - -SELECT - client, - COUNTIF(is_lcp_preloaded) AS lcp_preloaded, - COUNT(0) AS total, - COUNTIF(is_lcp_preloaded) / COUNT(0) AS pct_lcp_preloaded -FROM - preloaded -GROUP BY - client diff --git a/sql/2023/lcp_preload_discoverable.sql b/sql/2023/lcp_preload_discoverable.sql index 8e800a7c993..8641e593751 100644 --- a/sql/2023/lcp_preload_discoverable.sql +++ b/sql/2023/lcp_preload_discoverable.sql @@ -1,10 +1,13 @@ WITH lcp AS ( SELECT - _TABLE_SUFFIX AS client, - JSON_VALUE(payload, '$._performance.is_lcp_statically_discoverable') = 'true' AS discoverable, - JSON_VALUE(payload, '$._performance.is_lcp_preloaded') = 'true' AS preloaded + client, + JSON_VALUE(custom_metrics, '$.performance.is_lcp_statically_discoverable') = 'true' AS discoverable, + JSON_VALUE(custom_metrics, '$.performance.is_lcp_preloaded') = 'true' AS preloaded FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/lcp_resource_delay.sql b/sql/2023/lcp_resource_load_delay.sql similarity index 51% rename from sql/2023/lcp_resource_delay.sql rename to sql/2023/lcp_resource_load_delay.sql index c034510b258..19221b658ed 100644 --- a/sql/2023/lcp_resource_delay.sql +++ b/sql/2023/lcp_resource_load_delay.sql @@ -1,27 +1,33 @@ WITH pages AS ( SELECT - _TABLE_SUFFIX AS client, - url AS page, - JSON_VALUE(payload, '$._performance.lcp_elem_stats.url') AS url, + client, + page, + JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url, httparchive.core_web_vitals.GET_LAB_TTFB(payload) AS ttfb FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ), requests AS ( SELECT - _TABLE_SUFFIX AS client, + client, page, url, CAST(JSON_QUERY(payload, '$._created') AS FLOAT64) AS lcp_req_time FROM - `httparchive.requests.2022_06_01_*` + `httparchive.all.requests` + WHERE + date = '2023-10-01' AND + is_root_page ), delays AS ( SELECT client, - CAST(lcp_req_time - ttfb AS INT64) AS lcp_resource_delay + CAST(lcp_req_time - ttfb AS INT64) AS lcp_resource_load_delay FROM pages JOIN @@ -29,15 +35,13 @@ delays AS ( USING (client, page, url) WHERE - lcp_req_time IS NOT NULL AND - lcp_req_time > 0 AND - ttfb IS NOT NULL + lcp_req_time > ttfb ) SELECT percentile, client, - APPROX_QUANTILES(lcp_resource_delay, 1000)[OFFSET(percentile * 10)] AS lcp_resource_delay + APPROX_QUANTILES(lcp_resource_load_delay, 1000)[OFFSET(percentile * 10)] AS lcp_resource_load_delay FROM delays, UNNEST([10, 25, 50, 75, 90]) AS percentile diff --git a/sql/2023/lcp_resource_type.sql b/sql/2023/lcp_resource_type.sql index fae34d9e39f..06d7471cc57 100644 --- a/sql/2023/lcp_resource_type.sql +++ b/sql/2023/lcp_resource_type.sql @@ -1,11 +1,14 @@ WITH lcp AS ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, # Parse anchors out of LCP URLs. - REGEXP_EXTRACT(JSON_VALUE(payload, '$._performance.lcp_elem_stats.url'), r'([^#]*)') AS url + REGEXP_EXTRACT(JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url'), r'([^#]*)') AS url FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2023-10-01' AND + is_root_page ) diff --git a/sql/2023/lcp_responsive_data.sql b/sql/2023/lcp_responsive_data.sql index 8077dcb37b5..951023dd6f5 100644 --- a/sql/2023/lcp_responsive_data.sql +++ b/sql/2023/lcp_responsive_data.sql @@ -1,7 +1,7 @@ CREATE TEMP FUNCTION checkResponsiveImages(responsivelist STRING, lcpImgUrl STRING, nodePath STRING) RETURNS BOOLEAN LANGUAGE js AS ''' try { //we will check lcp elment is img - const lastSegment = (JSON.parse(nodePath).split(",").reverse())[0]; + const lastSegment = (nodePath.split(",").reverse())[0]; let lastNodeImg = false if(lastSegment == 'IMG'){ lastNodeImg = true @@ -11,7 +11,7 @@ CREATE TEMP FUNCTION checkResponsiveImages(responsivelist STRING, lcpImgUrl STRI if(listJson.length > 0){ for(let i=0;i= 0.75) / COUNTIF(fast_lcp IS NOT NULL) AS pct_good_lcp, - COUNTIF(small_cls / (small_cls + medium_cls + large_cls) >= 0.75) / COUNTIF(small_cls IS NOT NULL) AS pct_good_cls -FROM - `chrome-ux-report.materialized.device_summary` -WHERE - date BETWEEN '2020-01-01' AND '2022-06-01' AND - device IN ('desktop', 'phone') -GROUP BY - date, - device -ORDER BY - date, - device diff --git a/sql/2023/prelcp_domain_sharding.sql b/sql/2023/prelcp_domain_sharding.sql deleted file mode 100644 index 7181d7785cb..00000000000 --- a/sql/2023/prelcp_domain_sharding.sql +++ /dev/null @@ -1,58 +0,0 @@ -WITH requests AS ( - SELECT - _TABLE_SUFFIX AS client, - page, - NET.HOST(url) AS host, - CAST(JSON_QUERY(payload, '$._all_start') AS INT64) AS req_start - FROM - `httparchive.requests.2022_06_01_*` -), - -lcp AS ( - SELECT - _TABLE_SUFFIX AS client, - url AS page, - httparchive.core_web_vitals.GET_LAB_LCP(payload) AS lcp_time - FROM - `httparchive.pages.2022_06_01_*` -), - -hosts AS ( - SELECT - client, - COUNT(DISTINCT host) AS pre_lcp_hosts, - ANY_VALUE(lcp_time) AS lcp - FROM - lcp - JOIN - requests - USING - (client, page) - WHERE - req_start <= lcp_time - GROUP BY - client, - page -) - -SELECT - percentile, - client, - CASE - WHEN lcp <= 2500 THEN 'good' - WHEN lcp > 4000 THEN 'poor' - ELSE 'needs improvement' - END AS lcp, - COUNT(0) AS pages, - APPROX_QUANTILES(pre_lcp_hosts, 1000)[OFFSET(percentile * 10)] AS pre_lcp_hosts -FROM - hosts, - UNNEST([10, 25, 50, 75, 90]) AS percentile -GROUP BY - percentile, - client, - lcp -ORDER BY - percentile, - client, - lcp diff --git a/sql/2023/render_blocking_resources.sql b/sql/2023/render_blocking_resources.sql deleted file mode 100644 index 508961af546..00000000000 --- a/sql/2023/render_blocking_resources.sql +++ /dev/null @@ -1,22 +0,0 @@ -WITH lh AS ( - SELECT - IF(STARTS_WITH(_TABLE_SUFFIX, '2021'), '2021', '2022') AS year, - IF(ENDS_WITH(_TABLE_SUFFIX, 'desktop'), 'desktop', 'mobile') AS client, - CAST(JSON_VALUE(report, '$.audits.render-blocking-resources.score') AS FLOAT64) >= 0.9 AS is_passing - FROM - `httparchive.lighthouse.*` - WHERE - _TABLE_SUFFIX IN ('2021_07_01_mobile', '2022_06_01_mobile', '2022_06_01_desktop') -) - -SELECT - year, - client, - COUNTIF(is_passing) AS is_passing, - COUNT(0) AS total, - COUNTIF(is_passing) / COUNT(0) AS pct_passing -FROM - lh -GROUP BY - year, - client diff --git a/sql/2023/ttfb_by_cms.sql b/sql/2023/ttfb_by_cms.sql deleted file mode 100644 index d2fc69c7105..00000000000 --- a/sql/2023/ttfb_by_cms.sql +++ /dev/null @@ -1,26 +0,0 @@ -SELECT - app, - client, - COUNT(DISTINCT origin) AS n, - SUM(IF(ttfb.start < 800, ttfb.density, 0)) / SUM(ttfb.density) AS fast, - SUM(IF(ttfb.start >= 800 AND ttfb.start < 1800, ttfb.density, 0)) / SUM(ttfb.density) AS avg, - SUM(IF(ttfb.start >= 1800, ttfb.density, 0)) / SUM(ttfb.density) AS slow -FROM - `chrome-ux-report.all.202206`, - UNNEST(experimental.time_to_first_byte.histogram.bin) AS ttfb -JOIN ( - SELECT - _TABLE_SUFFIX AS client, - * - FROM - `httparchive.technologies.2022_06_01_*` - WHERE - category = 'CMS') -ON - client = IF(form_factor.name = 'desktop', 'desktop', 'mobile') AND - CONCAT(origin, '/') = url -GROUP BY - app, - client -ORDER BY - n DESC diff --git a/sql/2023/ttfb_by_rendering.sql b/sql/2023/ttfb_by_rendering.sql deleted file mode 100644 index c0ddd2ca935..00000000000 --- a/sql/2023/ttfb_by_rendering.sql +++ /dev/null @@ -1,20 +0,0 @@ -CREATE TEMPORARY FUNCTION getCSRRatio(payload STRING) RETURNS FLOAT64 LANGUAGE js AS ''' -try { - var $ = JSON.parse(payload); - var stats = JSON.parse($._wpt_bodies); - return (stats.visible_words.rendered / stats.visible_words.raw) || null; -} catch (e) { - return null; -} -'''; - -SELECT - _TABLE_SUFFIX AS client, - IF(getCSRRatio(payload) > 1.5, 'CSR', 'SSR') AS rendering, - COUNT(0) AS pages, - APPROX_QUANTILES(httparchive.core_web_vitals.GET_CRUX_TTFB(payload), 1000 IGNORE NULLS)[OFFSET(500)] AS median_ttfb -FROM - `httparchive.pages.2022_06_01_*` -GROUP BY - client, - rendering diff --git a/sql/2023/viewport_meta_zoom_disable.sql b/sql/2023/viewport_meta_zoom_disable.sql index 8b289a910c6..dd2b429f0fc 100644 --- a/sql/2023/viewport_meta_zoom_disable.sql +++ b/sql/2023/viewport_meta_zoom_disable.sql @@ -1,6 +1,12 @@ SELECT - COUNTIF(JSON_VALUE(report, '$.audits.viewport.score') = '0') AS viewport_failed, + client, + COUNTIF(JSON_VALUE(lighthouse, '$.audits.viewport.score') = '0') AS viewport_failed, COUNT(0) AS total, - COUNTIF(JSON_VALUE(report, '$.audits.viewport.score') = '0') / COUNT(0) AS pct_failed + COUNTIF(JSON_VALUE(lighthouse, '$.audits.viewport.score') = '0') / COUNT(0) AS pct_failed FROM - `httparchive.lighthouse.2022_06_01_mobile` + `httparchive.all.pages` +WHERE + date = '2023-10-01' AND + is_root_page +GROUP BY + client diff --git a/sql/2023/web_vitals_by_country.sql b/sql/2023/web_vitals_by_country.sql deleted file mode 100644 index 1a7e55424f2..00000000000 --- a/sql/2023/web_vitals_by_country.sql +++ /dev/null @@ -1,176 +0,0 @@ -#standardSQL -# Core WebVitals by country - -CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 -); - -CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 -); - -CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - NOT IS_GOOD(good, needs_improvement, poor) AND - NOT IS_POOR(good, needs_improvement, poor) -); - -CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); - -WITH -base AS ( - SELECT - origin, - country_code, - - SUM(fast_fid) / SUM(fast_fid + avg_fid + slow_fid) AS fast_fid, - SUM(avg_fid) / SUM(fast_fid + avg_fid + slow_fid) AS avg_fid, - SUM(slow_fid) / SUM(fast_fid + avg_fid + slow_fid) AS slow_fid, - - SUM(fast_inp) / SUM(fast_inp + avg_inp + slow_inp) AS fast_inp, - SUM(avg_inp) / SUM(fast_inp + avg_inp + slow_inp) AS avg_inp, - SUM(slow_inp) / SUM(fast_inp + avg_inp + slow_inp) AS slow_inp, - - SUM(fast_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS fast_lcp, - SUM(avg_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS avg_lcp, - SUM(slow_lcp) / SUM(fast_lcp + avg_lcp + slow_lcp) AS slow_lcp, - - SUM(small_cls) / SUM(small_cls + medium_cls + large_cls) AS small_cls, - SUM(medium_cls) / SUM(small_cls + medium_cls + large_cls) AS medium_cls, - SUM(large_cls) / SUM(small_cls + medium_cls + large_cls) AS large_cls, - - SUM(fast_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS fast_fcp, - SUM(avg_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS avg_fcp, - SUM(slow_fcp) / SUM(fast_fcp + avg_fcp + slow_fcp) AS slow_fcp, - - SUM(fast_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS fast_ttfb, - SUM(avg_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS avg_ttfb, - SUM(slow_ttfb) / SUM(fast_ttfb + avg_ttfb + slow_ttfb) AS slow_ttfb - - FROM - `chrome-ux-report.materialized.country_summary` - WHERE - yyyymm = 202206 - GROUP BY - origin, - country_code -) - -SELECT - `chrome-ux-report`.experimental.GET_COUNTRY(country_code) AS country, - - COUNT(DISTINCT origin) AS total_origins, - - # Good CWV with optional FID - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor - -FROM - base -GROUP BY - country -ORDER BY - total_origins DESC diff --git a/sql/2023/web_vitals_by_device.sql b/sql/2023/web_vitals_by_device.sql index 59baec43897..5e03fc42aba 100644 --- a/sql/2023/web_vitals_by_device.sql +++ b/sql/2023/web_vitals_by_device.sql @@ -1,6 +1,3 @@ -#standardSQL -# Core WebVitals by device - CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 ); @@ -53,7 +50,7 @@ base AS ( `chrome-ux-report.materialized.device_summary` WHERE device IN ('desktop', 'phone') AND - date IN ('2020-08-01', '2021-07-01', '2022-06-01') + date IN ('2020-08-01', '2021-07-01', '2022-06-01', '2023-09-01') ) SELECT @@ -70,7 +67,17 @@ SELECT IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv23_good, + + # Good CWV with optional INP + SAFE_DIVIDE( + COUNT(DISTINCT IF( + IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND + IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND + IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), + COUNT(DISTINCT IF( + IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv24_good, SAFE_DIVIDE( COUNT(DISTINCT IF( diff --git a/sql/2023/web_vitals_by_eff_connection_type.sql b/sql/2023/web_vitals_by_eff_connection_type.sql deleted file mode 100644 index 0864b01956a..00000000000 --- a/sql/2023/web_vitals_by_eff_connection_type.sql +++ /dev/null @@ -1,309 +0,0 @@ -#standardSQL -# WebVitals by effective connection type - -CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 -); - -CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(good, (good + needs_improvement + poor)) < 0.75 AND - SAFE_DIVIDE(poor, (good + needs_improvement + poor)) < 0.25 -); - -CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 -); - -CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); - -WITH -base AS ( - SELECT - origin, - effective_connection_type.name AS network, - layout_instability, - largest_contentful_paint, - first_input, - first_contentful_paint, - experimental.time_to_first_byte AS time_to_first_byte, - experimental.interaction_to_next_paint AS interaction_to_next_paint - FROM - `chrome-ux-report.all.202206` -), - -cls AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 0.1, bin.density, 0)) AS small, - SUM(IF(bin.start > 0.1 AND bin.start < 0.25, bin.density, 0)) AS medium, - SUM(IF(bin.start >= 0.25, bin.density, 0)) AS large, - `chrome-ux-report`.experimental.PERCENTILE_NUMERIC(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(layout_instability.cumulative_layout_shift.histogram.bin) AS bin - GROUP BY - origin, - network -), - -lcp AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 2500, bin.density, 0)) AS fast, - SUM(IF(bin.start >= 2500 AND bin.start < 4000, bin.density, 0)) AS avg, - SUM(IF(bin.start >= 4000, bin.density, 0)) AS slow, - `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(largest_contentful_paint.histogram.bin) AS bin - GROUP BY - origin, - network -), - -fid AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 100, bin.density, 0)) AS fast, - SUM(IF(bin.start >= 100 AND bin.start < 300, bin.density, 0)) AS avg, - SUM(IF(bin.start >= 300, bin.density, 0)) AS slow, - `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(first_input.delay.histogram.bin) AS bin - GROUP BY - origin, - network -), - -inp AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 200, bin.density, 0)) AS fast, - SUM(IF(bin.start >= 200 AND bin.start < 500, bin.density, 0)) AS avg, - SUM(IF(bin.start >= 500, bin.density, 0)) AS slow, - `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(interaction_to_next_paint.histogram.bin) AS bin - GROUP BY - origin, - network -), - -fcp AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 1800, bin.density, 0)) AS fast, - SUM(IF(bin.start >= 1800 AND bin.start < 3000, bin.density, 0)) AS avg, - SUM(IF(bin.start >= 3000, bin.density, 0)) AS slow, - `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(first_contentful_paint.histogram.bin) AS bin - GROUP BY - origin, - network -), - -ttfb AS ( - SELECT - origin, - network, - SUM(IF(bin.start < 500, bin.density, 0)) AS fast, - SUM(IF(bin.start >= 500 AND bin.start < 1500, bin.density, 0)) AS avg, - SUM(IF(bin.start >= 1500, bin.density, 0)) AS slow, - `chrome-ux-report`.experimental.PERCENTILE(ARRAY_AGG(bin), 75) AS p75 - FROM - base - LEFT JOIN - UNNEST(time_to_first_byte.histogram.bin) AS bin - GROUP BY - origin, - network -), - -granular_metrics AS ( - SELECT - origin, - network, - cls.small AS small_cls, - cls.medium AS medium_cls, - cls.large AS large_cls, - cls.p75 AS p75_cls, - - lcp.fast AS fast_lcp, - lcp.avg AS avg_lcp, - lcp.slow AS slow_lcp, - lcp.p75 AS p75_lcp, - - fid.fast AS fast_fid, - fid.avg AS avg_fid, - fid.slow AS slow_fid, - fid.p75 AS p75_fid, - - inp.fast AS fast_inp, - inp.avg AS avg_inp, - inp.slow AS slow_inp, - inp.p75 AS p75_inp, - - fcp.fast AS fast_fcp, - fcp.avg AS avg_fcp, - fcp.slow AS slow_fcp, - fcp.p75 AS p75_fcp, - - ttfb.fast AS fast_ttfb, - ttfb.avg AS avg_ttfb, - ttfb.slow AS slow_ttfb, - ttfb.p75 AS p75_ttfb - FROM - cls - LEFT JOIN - lcp - USING - (origin, network) - LEFT JOIN - fid - USING - (origin, network) - LEFT JOIN - inp - USING - (origin, network) - LEFT JOIN - fcp - USING - (origin, network) - LEFT JOIN - ttfb - USING - (origin, network) -) - -SELECT - network, - - COUNT(DISTINCT origin) AS total_origins, - - # Good CWV with optional FID - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor - -FROM - granular_metrics -GROUP BY - network diff --git a/sql/2023/web_vitals_by_rank.sql b/sql/2023/web_vitals_by_rank.sql deleted file mode 100644 index 81241233ba3..00000000000 --- a/sql/2023/web_vitals_by_rank.sql +++ /dev/null @@ -1,182 +0,0 @@ -#standardSQL -# Core WebVitals by rank - -CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 -); - -CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 -); - -CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - NOT IS_GOOD(good, needs_improvement, poor) AND - NOT IS_POOR(good, needs_improvement, poor) -); - -CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); - -WITH -base AS ( - SELECT - date, - origin, - rank, - - fast_fid, - avg_fid, - slow_fid, - - fast_inp, - avg_inp, - slow_inp, - - fast_lcp, - avg_lcp, - slow_lcp, - - small_cls, - medium_cls, - large_cls, - - fast_fcp, - avg_fcp, - slow_fcp, - - fast_ttfb, - avg_ttfb, - slow_ttfb - - FROM - `chrome-ux-report.materialized.metrics_summary` - WHERE - date IN ('2022-06-01') -) - -SELECT - date, - CASE - WHEN rank_grouping = 10000000 THEN 'all' - ELSE CAST(rank_grouping AS STRING) - END AS ranking, - - COUNT(DISTINCT origin) AS total_origins, - - # Good CWV with optional FID - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor - -FROM - base, - UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping -WHERE - rank <= rank_grouping -GROUP BY - date, - rank_grouping -ORDER BY - rank_grouping diff --git a/sql/2023/web_vitals_by_rank_and_device.sql b/sql/2023/web_vitals_by_rank_and_device.sql index efbdd9fa0fd..0d7d14bc6ae 100644 --- a/sql/2023/web_vitals_by_rank_and_device.sql +++ b/sql/2023/web_vitals_by_rank_and_device.sql @@ -1,6 +1,3 @@ -#standardSQL -# Core WebVitals by rank and device - CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 ); @@ -54,7 +51,7 @@ base AS ( `chrome-ux-report.materialized.device_summary` WHERE device IN ('desktop', 'phone') AND - date IN ('2022-06-01') + date IN ('2022-06-01', '2023-09-01') ) SELECT @@ -72,9 +69,9 @@ SELECT IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv23_good, - # Good CWV with optional INP (hypothetical!) + # Good CWV with optional INP SAFE_DIVIDE( COUNT(DISTINCT IF( IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND @@ -82,7 +79,7 @@ SELECT IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), COUNT(DISTINCT IF( IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_inp_good, + IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv24_good, SAFE_DIVIDE( COUNT(DISTINCT IF( @@ -182,7 +179,7 @@ SELECT FROM base, - UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS rank_grouping WHERE rank <= rank_grouping GROUP BY diff --git a/sql/2023/web_vitals_by_technology.sql b/sql/2023/web_vitals_by_technology.sql deleted file mode 100644 index f99e73300ea..00000000000 --- a/sql/2023/web_vitals_by_technology.sql +++ /dev/null @@ -1,208 +0,0 @@ -CREATE TEMP FUNCTION IS_GOOD (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(good, (good + needs_improvement + poor)) >= 0.75 -); - -CREATE TEMP FUNCTION IS_POOR (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - SAFE_DIVIDE(poor, (good + needs_improvement + poor)) >= 0.25 -); - -CREATE TEMP FUNCTION IS_NI (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - NOT IS_GOOD(good, needs_improvement, poor) AND - NOT IS_POOR(good, needs_improvement, poor) -); - -CREATE TEMP FUNCTION IS_NON_ZERO (good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( - good + needs_improvement + poor > 0 -); - -WITH base AS ( - SELECT - date, - origin, - CONCAT(origin, '/') AS page, - CASE - WHEN device = 'phone' THEN 'mobile' - ELSE device - END AS client, - - fast_fid, - avg_fid, - slow_fid, - - fast_inp, - avg_inp, - slow_inp, - - fast_lcp, - avg_lcp, - slow_lcp, - - small_cls, - medium_cls, - large_cls, - - fast_fcp, - avg_fcp, - slow_fcp, - - fast_ttfb, - avg_ttfb, - slow_ttfb - FROM - `chrome-ux-report.materialized.device_summary` - WHERE - device IN ('desktop', 'phone') AND - date IN ('2022-06-01') -), - -tech AS ( - SELECT - _TABLE_SUFFIX AS client, - category, - app AS technology, - url AS page - FROM - `httparchive.technologies.2022_06_01_*` - WHERE - category IN ('CMS', 'Ecommerce', 'JavaScript frameworks') -) - -SELECT - date, - client, - category, - technology, - - COUNT(DISTINCT origin) AS total_origins, - - # Good CWV with optional FID - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid) IS NOT FALSE AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_good, - - # Good CWV with optional INP (hypothetical!) - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_inp, avg_inp, slow_inp) IS NOT FALSE AND - IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AND - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AND - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cwv_inp_good, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_lcp, avg_lcp, slow_lcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp), origin, NULL))) AS pct_lcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fid, avg_fid, slow_fid), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fid, avg_fid, slow_fid), origin, NULL))) AS pct_fid_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_inp, avg_inp, slow_inp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_inp, avg_inp, slow_inp), origin, NULL))) AS pct_inp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(small_cls, medium_cls, large_cls), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(small_cls, medium_cls, large_cls), origin, NULL))) AS pct_cls_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_fcp, avg_fcp, slow_fcp), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp), origin, NULL))) AS pct_fcp_poor, - - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_good, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_NI(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_ni, - SAFE_DIVIDE( - COUNT(DISTINCT IF( - IS_POOR(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL)), - COUNT(DISTINCT IF( - IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb), origin, NULL))) AS pct_ttfb_poor - -FROM - base -JOIN - tech -USING - (client, page) -GROUP BY - date, - client, - category, - technology -HAVING - total_origins >= 1000 -ORDER BY - total_origins DESC From f8f74000103424f9137d1c093086ae84fc3005b3 Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Sun, 22 Oct 2023 20:08:58 -0400 Subject: [PATCH 3/7] 2023/perf --- sql/2023/{ => performance}/bfcache_cachecontrol_nostore.sql | 0 sql/2023/{ => performance}/bfcache_unload.sql | 0 sql/2023/{ => performance}/cls_animations.sql | 0 sql/2023/{ => performance}/cls_unsized_image_height.sql | 0 sql/2023/{ => performance}/cls_unsized_images.sql | 0 sql/2023/{ => performance}/inp_long_tasks.sql | 0 sql/2023/{ => performance}/inp_tbt.sql | 0 sql/2023/{ => performance}/js_bytes_rank.sql | 0 sql/2023/{ => performance}/lcp_bytes_distribution.sql | 0 sql/2023/{ => performance}/lcp_bytes_histogram.sql | 0 sql/2023/{ => performance}/lcp_element_data.sql | 0 sql/2023/{ => performance}/lcp_format.sql | 0 sql/2023/{ => performance}/lcp_host.sql | 0 sql/2023/{ => performance}/lcp_host_3p.sql | 0 sql/2023/{ => performance}/lcp_initiator_type.sql | 0 sql/2023/{ => performance}/lcp_lazy.sql | 0 sql/2023/{ => performance}/lcp_lazy_technologies.sql | 0 sql/2023/{ => performance}/lcp_preload_discoverable.sql | 0 sql/2023/{ => performance}/lcp_resource_load_delay.sql | 0 sql/2023/{ => performance}/lcp_resource_type.sql | 0 sql/2023/{ => performance}/lcp_responsive_data.sql | 0 sql/2023/{ => performance}/lcp_wasted_bytes.sql | 0 sql/2023/{ => performance}/viewport_meta_zoom_disable.sql | 0 sql/2023/{ => performance}/web_vitals_by_device.sql | 0 sql/2023/{ => performance}/web_vitals_by_rank_and_device.sql | 0 25 files changed, 0 insertions(+), 0 deletions(-) rename sql/2023/{ => performance}/bfcache_cachecontrol_nostore.sql (100%) rename sql/2023/{ => performance}/bfcache_unload.sql (100%) rename sql/2023/{ => performance}/cls_animations.sql (100%) rename sql/2023/{ => performance}/cls_unsized_image_height.sql (100%) rename sql/2023/{ => performance}/cls_unsized_images.sql (100%) rename sql/2023/{ => performance}/inp_long_tasks.sql (100%) rename sql/2023/{ => performance}/inp_tbt.sql (100%) rename sql/2023/{ => performance}/js_bytes_rank.sql (100%) rename sql/2023/{ => performance}/lcp_bytes_distribution.sql (100%) rename sql/2023/{ => performance}/lcp_bytes_histogram.sql (100%) rename sql/2023/{ => performance}/lcp_element_data.sql (100%) rename sql/2023/{ => performance}/lcp_format.sql (100%) rename sql/2023/{ => performance}/lcp_host.sql (100%) rename sql/2023/{ => performance}/lcp_host_3p.sql (100%) rename sql/2023/{ => performance}/lcp_initiator_type.sql (100%) rename sql/2023/{ => performance}/lcp_lazy.sql (100%) rename sql/2023/{ => performance}/lcp_lazy_technologies.sql (100%) rename sql/2023/{ => performance}/lcp_preload_discoverable.sql (100%) rename sql/2023/{ => performance}/lcp_resource_load_delay.sql (100%) rename sql/2023/{ => performance}/lcp_resource_type.sql (100%) rename sql/2023/{ => performance}/lcp_responsive_data.sql (100%) rename sql/2023/{ => performance}/lcp_wasted_bytes.sql (100%) rename sql/2023/{ => performance}/viewport_meta_zoom_disable.sql (100%) rename sql/2023/{ => performance}/web_vitals_by_device.sql (100%) rename sql/2023/{ => performance}/web_vitals_by_rank_and_device.sql (100%) diff --git a/sql/2023/bfcache_cachecontrol_nostore.sql b/sql/2023/performance/bfcache_cachecontrol_nostore.sql similarity index 100% rename from sql/2023/bfcache_cachecontrol_nostore.sql rename to sql/2023/performance/bfcache_cachecontrol_nostore.sql diff --git a/sql/2023/bfcache_unload.sql b/sql/2023/performance/bfcache_unload.sql similarity index 100% rename from sql/2023/bfcache_unload.sql rename to sql/2023/performance/bfcache_unload.sql diff --git a/sql/2023/cls_animations.sql b/sql/2023/performance/cls_animations.sql similarity index 100% rename from sql/2023/cls_animations.sql rename to sql/2023/performance/cls_animations.sql diff --git a/sql/2023/cls_unsized_image_height.sql b/sql/2023/performance/cls_unsized_image_height.sql similarity index 100% rename from sql/2023/cls_unsized_image_height.sql rename to sql/2023/performance/cls_unsized_image_height.sql diff --git a/sql/2023/cls_unsized_images.sql b/sql/2023/performance/cls_unsized_images.sql similarity index 100% rename from sql/2023/cls_unsized_images.sql rename to sql/2023/performance/cls_unsized_images.sql diff --git a/sql/2023/inp_long_tasks.sql b/sql/2023/performance/inp_long_tasks.sql similarity index 100% rename from sql/2023/inp_long_tasks.sql rename to sql/2023/performance/inp_long_tasks.sql diff --git a/sql/2023/inp_tbt.sql b/sql/2023/performance/inp_tbt.sql similarity index 100% rename from sql/2023/inp_tbt.sql rename to sql/2023/performance/inp_tbt.sql diff --git a/sql/2023/js_bytes_rank.sql b/sql/2023/performance/js_bytes_rank.sql similarity index 100% rename from sql/2023/js_bytes_rank.sql rename to sql/2023/performance/js_bytes_rank.sql diff --git a/sql/2023/lcp_bytes_distribution.sql b/sql/2023/performance/lcp_bytes_distribution.sql similarity index 100% rename from sql/2023/lcp_bytes_distribution.sql rename to sql/2023/performance/lcp_bytes_distribution.sql diff --git a/sql/2023/lcp_bytes_histogram.sql b/sql/2023/performance/lcp_bytes_histogram.sql similarity index 100% rename from sql/2023/lcp_bytes_histogram.sql rename to sql/2023/performance/lcp_bytes_histogram.sql diff --git a/sql/2023/lcp_element_data.sql b/sql/2023/performance/lcp_element_data.sql similarity index 100% rename from sql/2023/lcp_element_data.sql rename to sql/2023/performance/lcp_element_data.sql diff --git a/sql/2023/lcp_format.sql b/sql/2023/performance/lcp_format.sql similarity index 100% rename from sql/2023/lcp_format.sql rename to sql/2023/performance/lcp_format.sql diff --git a/sql/2023/lcp_host.sql b/sql/2023/performance/lcp_host.sql similarity index 100% rename from sql/2023/lcp_host.sql rename to sql/2023/performance/lcp_host.sql diff --git a/sql/2023/lcp_host_3p.sql b/sql/2023/performance/lcp_host_3p.sql similarity index 100% rename from sql/2023/lcp_host_3p.sql rename to sql/2023/performance/lcp_host_3p.sql diff --git a/sql/2023/lcp_initiator_type.sql b/sql/2023/performance/lcp_initiator_type.sql similarity index 100% rename from sql/2023/lcp_initiator_type.sql rename to sql/2023/performance/lcp_initiator_type.sql diff --git a/sql/2023/lcp_lazy.sql b/sql/2023/performance/lcp_lazy.sql similarity index 100% rename from sql/2023/lcp_lazy.sql rename to sql/2023/performance/lcp_lazy.sql diff --git a/sql/2023/lcp_lazy_technologies.sql b/sql/2023/performance/lcp_lazy_technologies.sql similarity index 100% rename from sql/2023/lcp_lazy_technologies.sql rename to sql/2023/performance/lcp_lazy_technologies.sql diff --git a/sql/2023/lcp_preload_discoverable.sql b/sql/2023/performance/lcp_preload_discoverable.sql similarity index 100% rename from sql/2023/lcp_preload_discoverable.sql rename to sql/2023/performance/lcp_preload_discoverable.sql diff --git a/sql/2023/lcp_resource_load_delay.sql b/sql/2023/performance/lcp_resource_load_delay.sql similarity index 100% rename from sql/2023/lcp_resource_load_delay.sql rename to sql/2023/performance/lcp_resource_load_delay.sql diff --git a/sql/2023/lcp_resource_type.sql b/sql/2023/performance/lcp_resource_type.sql similarity index 100% rename from sql/2023/lcp_resource_type.sql rename to sql/2023/performance/lcp_resource_type.sql diff --git a/sql/2023/lcp_responsive_data.sql b/sql/2023/performance/lcp_responsive_data.sql similarity index 100% rename from sql/2023/lcp_responsive_data.sql rename to sql/2023/performance/lcp_responsive_data.sql diff --git a/sql/2023/lcp_wasted_bytes.sql b/sql/2023/performance/lcp_wasted_bytes.sql similarity index 100% rename from sql/2023/lcp_wasted_bytes.sql rename to sql/2023/performance/lcp_wasted_bytes.sql diff --git a/sql/2023/viewport_meta_zoom_disable.sql b/sql/2023/performance/viewport_meta_zoom_disable.sql similarity index 100% rename from sql/2023/viewport_meta_zoom_disable.sql rename to sql/2023/performance/viewport_meta_zoom_disable.sql diff --git a/sql/2023/web_vitals_by_device.sql b/sql/2023/performance/web_vitals_by_device.sql similarity index 100% rename from sql/2023/web_vitals_by_device.sql rename to sql/2023/performance/web_vitals_by_device.sql diff --git a/sql/2023/web_vitals_by_rank_and_device.sql b/sql/2023/performance/web_vitals_by_rank_and_device.sql similarity index 100% rename from sql/2023/web_vitals_by_rank_and_device.sql rename to sql/2023/performance/web_vitals_by_rank_and_device.sql From 74e79e8f6faa865ec7c175281804d3c07a9e7989 Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Sun, 22 Oct 2023 20:11:55 -0400 Subject: [PATCH 4/7] lint --- sql/2023/performance/lcp_bytes_distribution.sql | 2 +- sql/2023/performance/lcp_bytes_histogram.sql | 2 +- sql/2023/performance/lcp_element_data.sql | 4 ++-- sql/2023/performance/lcp_lazy.sql | 2 +- sql/2023/performance/lcp_lazy_technologies.sql | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/2023/performance/lcp_bytes_distribution.sql b/sql/2023/performance/lcp_bytes_distribution.sql index 43ccd956b46..80eef7ea052 100644 --- a/sql/2023/performance/lcp_bytes_distribution.sql +++ b/sql/2023/performance/lcp_bytes_distribution.sql @@ -7,7 +7,7 @@ WITH pages AS ( `httparchive.all.pages` WHERE date = '2023-10-01' AND - is_root_page + is_root_page ), requests AS ( diff --git a/sql/2023/performance/lcp_bytes_histogram.sql b/sql/2023/performance/lcp_bytes_histogram.sql index a03fff1b6e4..47be833f630 100644 --- a/sql/2023/performance/lcp_bytes_histogram.sql +++ b/sql/2023/performance/lcp_bytes_histogram.sql @@ -7,7 +7,7 @@ WITH pages AS ( `httparchive.all.pages` WHERE date = '2023-10-01' AND - is_root_page + is_root_page ), requests AS ( diff --git a/sql/2023/performance/lcp_element_data.sql b/sql/2023/performance/lcp_element_data.sql index a2ca420dc23..44859802e0b 100644 --- a/sql/2023/performance/lcp_element_data.sql +++ b/sql/2023/performance/lcp_element_data.sql @@ -82,7 +82,7 @@ WITH lcp_stats AS ( `httparchive.all.pages` WHERE date = '2023-10-01' AND - is_root_page + is_root_page ) SELECT @@ -115,7 +115,7 @@ JOIN ( `httparchive.all.pages` WHERE date = '2023-10-01' AND - is_root_page + is_root_page GROUP BY client) USING diff --git a/sql/2023/performance/lcp_lazy.sql b/sql/2023/performance/lcp_lazy.sql index d41948742ef..86d0a6fb8b4 100644 --- a/sql/2023/performance/lcp_lazy.sql +++ b/sql/2023/performance/lcp_lazy.sql @@ -25,7 +25,7 @@ WITH lcp_stats AS ( SELECT client, isLazyLoaded(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS native_lazy, - hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy, + hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy FROM `httparchive.all.pages` WHERE diff --git a/sql/2023/performance/lcp_lazy_technologies.sql b/sql/2023/performance/lcp_lazy_technologies.sql index ab3f49c3d87..78e863d94ee 100644 --- a/sql/2023/performance/lcp_lazy_technologies.sql +++ b/sql/2023/performance/lcp_lazy_technologies.sql @@ -26,7 +26,7 @@ WITH lazy_tech AS ( client, page, isLazyLoaded(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS native_lazy, - hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy, + hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy t.technology FROM `httparchive.all.pages`, From c20e633ef3cccd2134fc53e07afdff0437e732de Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Sun, 22 Oct 2023 20:20:26 -0400 Subject: [PATCH 5/7] lint --- sql/2023/performance/lcp_lazy_technologies.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/2023/performance/lcp_lazy_technologies.sql b/sql/2023/performance/lcp_lazy_technologies.sql index 78e863d94ee..40509f07e45 100644 --- a/sql/2023/performance/lcp_lazy_technologies.sql +++ b/sql/2023/performance/lcp_lazy_technologies.sql @@ -26,8 +26,8 @@ WITH lazy_tech AS ( client, page, isLazyLoaded(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS native_lazy, - hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy - t.technology + hasLazyHeuristics(JSON_EXTRACT(custom_metrics, '$.performance.lcp_elem_stats.attributes')) AS custom_lazy, + t.technology AS technology FROM `httparchive.all.pages`, UNNEST(technologies) AS t @@ -73,4 +73,3 @@ HAVING pct_either_lazy > 0.1 ORDER BY either_lazy DESC - From a26ece0d1128b8abff433fe10cd6d2380009749f Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Mon, 23 Oct 2023 19:52:50 -0400 Subject: [PATCH 6/7] fix initiator --- sql/2023/performance/lcp_initiator_type.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2023/performance/lcp_initiator_type.sql b/sql/2023/performance/lcp_initiator_type.sql index 73040c08e35..6021dfb1c5b 100644 --- a/sql/2023/performance/lcp_initiator_type.sql +++ b/sql/2023/performance/lcp_initiator_type.sql @@ -2,7 +2,7 @@ WITH lcp AS ( SELECT client, page, - JSON_VALUE(custom_metrics, '$.performance.lcp_elem_stats.url') AS url, + JSON_VALUE(custom_metrics, '$.performance.lcp_resource.initiator.url') AS url, JSON_VALUE(custom_metrics, '$.performance.is_lcp_statically_discoverable') = 'false' AS not_discoverable FROM `httparchive.all.pages` From c627d365c6b22ace58da4f14ce6b58b5fdde441d Mon Sep 17 00:00:00 2001 From: Rick Viscomi Date: Wed, 25 Oct 2023 08:24:55 -0400 Subject: [PATCH 7/7] null initiators --- sql/2023/performance/lcp_initiator_type.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2023/performance/lcp_initiator_type.sql b/sql/2023/performance/lcp_initiator_type.sql index 6021dfb1c5b..29596bbbf32 100644 --- a/sql/2023/performance/lcp_initiator_type.sql +++ b/sql/2023/performance/lcp_initiator_type.sql @@ -26,13 +26,13 @@ requests AS ( SELECT client, - type AS lcp_initiator_type, + IFNULL(type, 'unknown') AS lcp_initiator_type, COUNTIF(not_discoverable) AS pages, SUM(COUNT(0)) OVER (PARTITION BY client) AS total, COUNTIF(not_discoverable) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct FROM lcp -JOIN +LEFT JOIN requests USING (client, page, url)