diff --git a/build-system/tasks/visual-diff.js b/build-system/tasks/visual-diff.js index 539ccde00e00..0b88b578f202 100644 --- a/build-system/tasks/visual-diff.js +++ b/build-system/tasks/visual-diff.js @@ -455,6 +455,19 @@ async function snapshotWebpages(percy, page, webpages, config) { await verifyCssElements(page, url, webpage.forbidden_css, webpage.loading_incomplete_css, webpage.loading_complete_css); + if (webpage.loading_complete_delay_ms) { + if (typeof webpage.loading_complete_delay_ms !== 'number' && + webpage.loading_complete_delay_ms > 0) { + log('verbose', 'Waiting', + colors.cyan(webpage.loading_complete_delay_ms + 'ms'), + 'for loading to complete'); + await sleep(webpage.loading_complete_delay_ms || 0); + } else { + log('warning', 'Skipping unknown delay', + webpage.loading_complete_delay_ms); + } + } + if (webpage.enable_percy_javascript) { snapshotOptions.enableJavaScript = true; // Remove all scripts that have an external source, leaving only those diff --git a/examples/visual-tests/amp-story/amp-story-bookend.html b/examples/visual-tests/amp-story/amp-story-bookend.html index 8ada77ba0f2d..390708f15356 100644 --- a/examples/visual-tests/amp-story/amp-story-bookend.html +++ b/examples/visual-tests/amp-story/amp-story-bookend.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1.hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/amp-story-bookend.rtl.html b/examples/visual-tests/amp-story/amp-story-bookend.rtl.html index 07bda2a0b31c..9f4f61e15b44 100644 --- a/examples/visual-tests/amp-story/amp-story-bookend.rtl.html +++ b/examples/visual-tests/amp-story/amp-story-bookend.rtl.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1.hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/amp-story-consent.html b/examples/visual-tests/amp-story/amp-story-consent.html index 3c407275fe01..4d887095f54f 100644 --- a/examples/visual-tests/amp-story/amp-story-consent.html +++ b/examples/visual-tests/amp-story/amp-story-consent.html @@ -20,6 +20,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/amp-story-consent.rtl.html b/examples/visual-tests/amp-story/amp-story-consent.rtl.html index ac52c846d050..a5961afdd6f7 100644 --- a/examples/visual-tests/amp-story/amp-story-consent.rtl.html +++ b/examples/visual-tests/amp-story/amp-story-consent.rtl.html @@ -20,6 +20,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/amp-story-cta-layer.html b/examples/visual-tests/amp-story/amp-story-cta-layer.html index 0d6461e13385..9642bb73c25d 100644 --- a/examples/visual-tests/amp-story/amp-story-cta-layer.html +++ b/examples/visual-tests/amp-story/amp-story-cta-layer.html @@ -17,7 +17,7 @@ } button { - background-color: red; + background-color: blue; color: white; position: absolute; top: 50%; diff --git a/examples/visual-tests/amp-story/basic.html b/examples/visual-tests/amp-story/basic.html index 6aed46a60484..779abd21343c 100644 --- a/examples/visual-tests/amp-story/basic.html +++ b/examples/visual-tests/amp-story/basic.html @@ -19,6 +19,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1.hello-world { color: white; text-shadow: 3px 3px rgba(0, 0, 0, 0.5); diff --git a/examples/visual-tests/amp-story/basic.rtl.html b/examples/visual-tests/amp-story/basic.rtl.html index b29a10e9e5f5..c3680fa93b9a 100644 --- a/examples/visual-tests/amp-story/basic.rtl.html +++ b/examples/visual-tests/amp-story/basic.rtl.html @@ -19,6 +19,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1.hello-world { color: white; text-shadow: 3px 3px rgba(0, 0, 0, 0.5); diff --git a/examples/visual-tests/amp-story/embed-mode-1.html b/examples/visual-tests/amp-story/embed-mode-1.html index 87d3e5ecc637..957d31a093cd 100644 --- a/examples/visual-tests/amp-story/embed-mode-1.html +++ b/examples/visual-tests/amp-story/embed-mode-1.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/embed-mode-2.html b/examples/visual-tests/amp-story/embed-mode-2.html index c8343db118b5..01a9e5428f16 100644 --- a/examples/visual-tests/amp-story/embed-mode-2.html +++ b/examples/visual-tests/amp-story/embed-mode-2.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/info-dialog.html b/examples/visual-tests/amp-story/info-dialog.html index e0ee0f79af68..58999ddf337b 100644 --- a/examples/visual-tests/amp-story/info-dialog.html +++ b/examples/visual-tests/amp-story/info-dialog.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/info-dialog.rtl.html b/examples/visual-tests/amp-story/info-dialog.rtl.html index 394f00c64e9c..d2551c4220d9 100644 --- a/examples/visual-tests/amp-story/info-dialog.rtl.html +++ b/examples/visual-tests/amp-story/info-dialog.rtl.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/share-menu.html b/examples/visual-tests/amp-story/share-menu.html index e063068beb5a..6b332d65e6ce 100644 --- a/examples/visual-tests/amp-story/share-menu.html +++ b/examples/visual-tests/amp-story/share-menu.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/examples/visual-tests/amp-story/share-menu.rtl.html b/examples/visual-tests/amp-story/share-menu.rtl.html index 5edf75eb4177..f89f0f442cfa 100644 --- a/examples/visual-tests/amp-story/share-menu.rtl.html +++ b/examples/visual-tests/amp-story/share-menu.rtl.html @@ -18,6 +18,11 @@ font-family: sans-serif; } + .i-amphtml-story-spinner-container { + /* Hide the spinner for loading pages, since this causes flakes */ + display: none; + } + h1#hello-world { color: white; text-align: center; diff --git a/test/visual-diff/visual-tests b/test/visual-diff/visual-tests index 2da5b0bb1474..07ce5a7732c5 100644 --- a/test/visual-diff/visual-tests +++ b/test/visual-diff/visual-tests @@ -75,6 +75,11 @@ * ".another-loading-complete-css-class" * ], * + * // [optional] A duration of time (in milliseconds) to wait after all + * // other loading is complete, before taking a snapshot. Can be used to + * // allow CSS transitions or other effects to complete. + * "loading_complete_delay_ms": 1000, + * * // [optional] Experiments that must be enabled via cookies. * "experiments": [ * "amp-feature-one", @@ -192,20 +197,16 @@ "enable_percy_javascript": true }, { - "flaky": true, - // amp-story tests are very flaky. - // e.g., https://percy.io/ampproject/amphtml/builds/894594 "url": "examples/visual-tests/amp-story/basic.html", "name": "amp-story: basic", "viewport": {"width": 320, "height": 480}, "loading_complete_css": [ ".i-amphtml-story-loaded", "amp-story-page#page-2[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-fill.html", "name": "amp-story: Grid layer (fill)", "viewport": {"width": 320, "height": 480}, @@ -215,8 +216,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-vertical.html", "name": "amp-story: Grid layer (vertical)", "viewport": {"width": 320, "height": 480}, @@ -226,8 +225,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-horizontal.html", "name": "amp-story: Grid layer (horizontal)", "viewport": {"width": 320, "height": 480}, @@ -237,8 +234,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-thirds.html", "name": "amp-story: Grid layer (thirds)", "viewport": {"width": 320, "height": 480}, @@ -251,19 +246,16 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-cta-layer.html", "name": "amp-story: CTA layer", "viewport": {"width": 320, "height": 480}, "loading_complete_css": [ ".i-amphtml-story-loaded", "amp-story-page#the-one-with-the-cta-layer[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/embed-mode-1.html", "name": "amp-story: embed mode 1", "viewport": {"width": 320, "height": 480}, @@ -272,8 +264,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/embed-mode-2.html", "name": "amp-story: embed mode 2", "viewport": {"width": 320, "height": 480}, @@ -282,8 +272,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/share-menu.html", "name": "amp-story: share menu", "viewport": {"width": 320, "height": 480}, @@ -293,8 +281,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/info-dialog.html", "name": "amp-story: info dialog", "viewport": {"width": 320, "height": 480}, @@ -304,8 +290,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-bookend.html", "name": "amp-story: bookend", "viewport": {"width": 320, "height": 480}, @@ -315,8 +299,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-consent.html", "name": "amp-story: consent", "viewport": {"width": 320, "height": 480}, @@ -326,8 +308,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-unsupported-browser-layer.html", "name": "amp-story: unsupported browser", "viewport": {"width": 320, "height": 480}, @@ -337,52 +317,47 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/basic.html", "name": "amp-story: basic (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", "amp-story-page#page-2[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-fill.html", "name": "amp-story: Grid layer (fill) (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-grid-template-fill" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-vertical.html", "name": "amp-story: Grid layer (vertical) (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-grid-template-vertical" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-horizontal.html", "name": "amp-story: Grid layer (horizontal) (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-grid-template-horizontal" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-grid-layer-template-thirds.html", "name": "amp-story: Grid layer (thirds) (desktop)", "viewport": {"width": 1440, "height": 900}, @@ -391,23 +366,22 @@ ], "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-grid-template-thirds" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-cta-layer.html", "name": "amp-story: CTA layer (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", "amp-story-page#the-one-with-the-cta-layer[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/embed-mode-1.html", "name": "amp-story: embed mode 1 (desktop)", "viewport": {"width": 1440, "height": 900}, @@ -416,40 +390,35 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/embed-mode-2.html", "name": "amp-story: embed mode 2 (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ - ".i-amphtml-story-loaded" + ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-bookend.html", "name": "amp-story: bookend (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-bookend" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-consent.html", "name": "amp-story: consent (desktop)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-consent" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-unsupported-browser-layer.html", "name": "amp-story: unsupported browser (desktop)", "viewport": {"width": 1440, "height": 900}, @@ -459,19 +428,16 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/basic.rtl.html", "name": "amp-story: basic (rtl)", "viewport": {"width": 320, "height": 480}, "loading_complete_css": [ ".i-amphtml-story-loaded", "amp-story-page#page-2[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/share-menu.rtl.html", "name": "amp-story: share menu (rtl)", "viewport": {"width": 320, "height": 480}, @@ -481,8 +447,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/info-dialog.rtl.html", "name": "amp-story: info dialog (rtl)", "viewport": {"width": 320, "height": 480}, @@ -492,8 +456,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-bookend.rtl.html", "name": "amp-story: bookend (rtl)", "viewport": {"width": 320, "height": 480}, @@ -503,8 +465,6 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-consent.rtl.html", "name": "amp-story: consent (rtl)", "viewport": {"width": 320, "height": 480}, @@ -514,41 +474,37 @@ ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/basic.rtl.html", "name": "amp-story: basic (desktop) (rtl)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", "amp-story-page#page-2[active]" - ] + ], + "loading_complete_delay_ms": 500, /* for page navigation */ }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-bookend.rtl.html", "name": "amp-story: bookend (desktop) (rtl)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-bookend" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. "url": "examples/visual-tests/amp-story/amp-story-consent.rtl.html", "name": "amp-story: consent (desktop) (rtl)", "viewport": {"width": 1440, "height": 900}, "loading_complete_css": [ ".i-amphtml-story-loaded", + ".i-amphtml-story-share-pill-label", ".i-amphtml-story-consent" ] }, { - "flaky": true, - // amp-story tests are very flaky. see above. // The url here should not matter; this UI should be triggered for all // story documents, based on the viewport dimensions. "url": "examples/visual-tests/amp-story/basic.html",