Merge pull request #167 from woocommerce/24-05/debug-self-tests

Upload test artifacts to debug self-tests flakiness

.github/workflows/qit-self-test-custom-test.yml

@@ -44,3 +44,10 @@ jobs:
       - name: Run tests in parallel
         working-directory: _tests/custom_tests
         run: ./vendor/bin/phpunit
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: qit-results
+          path: ${{ github.workspace }}/tmp/qit-results-*

.github/workflows/self-test-template.yml

@@ -62,14 +62,14 @@ jobs:
         working-directory: _tests/managed_tests
         run: composer install
 
-      #- name: Connect to QIT
-      #  run: ./qit partner:add --user="${{ secrets.QIT_USER }}" --application_password="${{ secrets.QIT_ACCESS_TOKEN }}"
+      - name: Connect to QIT
+        run: ./qit partner:add --user="${{ secrets.QIT_USER }}" --application_password="${{ secrets.QIT_ACCESS_TOKEN }}"
 
-      - name: Enable dev (Staging)
-        run: ./qit dev
+      #- name: Enable dev (Staging)
+      #  run: ./qit dev
 
-      - name: Connect to QIT (Staging)
-        run: ./qit backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
+      #- name: Connect to QIT (Staging)
+      #  run: ./qit backend:add --environment="staging" --qit_secret="${{ secrets.QIT_STAGING_SECRET }}" --manager_url="https://stagingcompatibilitydashboard.wpcomstaging.com"
 
       ### Cache ZIP Image
       - name: Set up date

.gitignore

@@ -30,3 +30,7 @@ _tests/custom_tests/.env
 _tests/custom_tests/tmp_qit_config*
 _tests/custom_tests/cache
 _tests/custom_tests/tmp/
+
+_tests/custom_tests/.env.staging
+
+_tests/custom_tests/.env.local

_tests/custom_tests/bootstrap.php

@@ -31,7 +31,7 @@ function qit( array $command, array $qit_env_json = [], int $expected_exit_code
 	 *
 	 * If we receive another command like the previous one while it's locked, wait.
 	 */
-	if ( 'tag:upload' === $command[0] ) {
+	if ( $command[0] === 'tag:upload' ) {
 		$lock_name = $command[1];
 		$lock_file = sprintf( '%s/qit-test-tag-lock-%s', sys_get_temp_dir(), md5( $lock_name ) );
 		$max_wait  = 60;
@@ -44,7 +44,7 @@ function qit( array $command, array $qit_env_json = [], int $expected_exit_code
 			}
 		}
 		touch( $lock_file );
-	} elseif ( 'tag:delete' === $command[0] ) {
+	} elseif ( $command[0] === 'tag:delete' ) {
 		$lock_name = $command[1];
 		$lock_file = sprintf( '%s/qit-test-tag-lock-%s', sys_get_temp_dir(), md5( $lock_name ) );
 		if ( file_exists( $lock_file ) ) {
@@ -58,6 +58,10 @@ function qit( array $command, array $qit_env_json = [], int $expected_exit_code
 		$args[] = '--config';
 		$args[] = $qit_config_filename;
 	}
+	if ( $command[0] === 'run:e2e' ) {
+		$args[] = '--pw_options';
+		$args[] = '"--trace on"';
+	}
 	$qit = new Process( $args );
 	$qit->setTimeout( 300 );
 	$qit->setIdleTimeout( 300 );

_tests/custom_tests/tests/EnvTest.php

@@ -101,35 +101,17 @@ public function test_env_up_with_plugins() {
 			'env:exec',
 			'--env_id',
 			$json['env_id'],
-			'wp plugin list',
+			'wp plugin list --fields=name,status',
 		] );
 
 		/**
-		 * name    status    update    version    update_version    auto_update
-		 * automatewoo    active    none    6.0.20        off
-		 * woocommerce    active    available    8.7.0.20    8.8.2    off
-		 * qit-wp-cli    must-use    none            off
-		 * wp-cli-github-cache    must-use    none            off
+		 * name    status
+		 * automatewoo    active
+		 * woocommerce    active
+		 * qit-wp-cli    must-use
+		 * wp-cli-github-cache    must-use
 		 */
 
-		// Iterate over each line, for the "automatewoo" and "woocommerce", normalize the version.
-		$lines                = explode( "\n", $output );
-		$headers              = preg_split( '/\s+/', trim( $lines[0] ) );  // Split the header to find the index of 'version'
-		$version_index        = array_search( 'version', $headers );  // Locate the index of the 'version' column
-		$update_version_index = array_search( 'update_version', $headers );  // Locate the index of the 'update_version' column
-		$update_index         = array_search( 'update', $headers );  // Locate the index of the 'update' column
-
-		foreach ( $lines as $key => $line ) {
-			if ( strpos( $line, 'automatewoo' ) !== false || strpos( $line, 'woocommerce' ) !== false ) {
-				$parts                          = preg_split( '/\s+/', trim( $line ) );
-				$parts[ $version_index ]        = 'NORMALIZED_VERSION';
-				$parts[ $update_version_index ] = 'NORMALIZED_VERSION';
-				$parts[ $update_index ]         = 'NORMALIZED';
-				$lines[ $key ]                  = implode( '    ', $parts );
-			}
-		}
-		$output = implode( "\n", $lines );
-
 		$this->assertMatchesNormalizedSnapshot( $output );
 	}
 

_tests/custom_tests/tests/Traits/ScaffoldHelpers.php

@@ -56,8 +56,9 @@ protected function normalize_scaffolded_test_run_output( string $output ): strin
 		 * Shutting down environment...
 		 */
 
-		// Normalize "1 passed (5.7s)" => "1 passed (TIME)"
-		$output = preg_replace( '/passed \(\d+\.\ds\)/', 'passed (TIME)', $output );
+		// Normalize "1 passed (5.7s)" => "passed (TIME)"
+		// Normalize "1 passed (1.0m)" => "passed (TIME)"
+		$output = preg_replace( '/passed \(\d+\.\d+[sm]\)/', 'passed (TIME)', $output );
 
 		// Normalize npm version, "10.2.4 -> 10.6.0" => "VERSION_1 -> VERSION_2"
 		$output = preg_replace( '/New minor version of npm available! \d+\.\d+\.\d+ -> \d+\.\d+\.\d+/', 'New minor version of npm available! VERSION_1 -> VERSION_2', $output );

_tests/custom_tests/tests/Traits/SnapshotHelpers.php

@@ -10,6 +10,7 @@ trait SnapshotHelpers {
 	public function assertMatchesNormalizedSnapshot( string $actual, ?\Spatie\Snapshots\Driver $driver = null ): void {
 		$actual = str_replace( sys_get_temp_dir(), '/tmp-normalized', $actual );
 		$actual = str_replace( '/tmp/', '/tmp-normalized/', $actual );
+		$actual = preg_replace( '/qit-results-[a-z0-9]+/', 'qit-results-normalizedid', $actual );
 
 		/*
 		 * "paratest" sets the "TEST_TOKEN" env var.

_tests/custom_tests/tests/__snapshots__/CompatibilityTest__test_multiple_tags_and_multiple_plugins_with_multiple_tags__1.txt

@@ -15,6 +15,7 @@ Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-multiple-test-tag
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-multiple-test-tags-another/bootstrap/bootstrap.sh
 Exporting DB
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 7 tests using 1 worker
 [1/7] [woocommerce-self-test-multiple-test-tags] › woocommerce/self-test-multiple-test-tags/example.spec.js:9:5 › I can see my plugin menu
 [2/7] [db-import] › db-import.js:5:5 › Database Import

_tests/custom_tests/tests/__snapshots__/CompatibilityTest__test_sut_and_activate_additional__1.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.php
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 1 test using 1 worker
 [1/1] [automatewoo-local] › automatewoo/local/example.spec.js:9:5 › I can see my plugin menu
 1 passed (TIME)

_tests/custom_tests/tests/__snapshots__/CompatibilityTest__test_sut_and_bootstrap_additional__1.txt

@@ -10,6 +10,7 @@ Bootstrapping woocommerce /qit/tests/e2e/woocommerce/self-test-bootstrap-additio
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.php
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 1 test using 1 worker
 [1/1] [automatewoo-local] › automatewoo/local/example.spec.js:9:5 › I can see my plugin menu
 1 passed (TIME)

_tests/custom_tests/tests/__snapshots__/CompatibilityTest__test_sut_and_test_additional__1.txt

@@ -11,6 +11,7 @@ Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.p
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.sh
 Exporting DB
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 3 tests using 1 worker
 [1/3] [woocommerce-self-test-sut-and-test-additional] › woocommerce/self-test-sut-and-test-additional/example.spec.js:9:5 › I can see my plugin menu
 [2/3] [db-import] › db-import.js:5:5 › Database Import

_tests/custom_tests/tests/__snapshots__/EnvTest__test_env_up_with_plugins__1.txt

@@ -1,6 +1,6 @@
-name	status	update	version	update_version	auto_update
-automatewoo    active    NORMALIZED    NORMALIZED_VERSION    NORMALIZED_VERSION
-woocommerce    active    NORMALIZED    NORMALIZED_VERSION    NORMALIZED_VERSION
-qit-wp-cli	must-use	none			off
-wp-cli-github-cache	must-use	none			off
+name	status
+automatewoo	active
+woocommerce	active
+qit-wp-cli	must-use
+wp-cli-github-cache	must-use
 

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_multiple_tags_and_run_tests__1.txt

@@ -11,6 +11,7 @@ Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-multiple-test-tag
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-multiple-test-tags-another/bootstrap/bootstrap.sh
 Exporting DB
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 3 tests using 1 worker
 [1/3] [automatewoo-self-test-multiple-test-tags] › automatewoo/self-test-multiple-test-tags/example.spec.js:9:5 › I can see my plugin menu
 [2/3] [db-import] › db-import.js:5:5 › Database Import

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_playwright_config_override__1.txt

@@ -10,6 +10,7 @@ Bootstrapping Plugins
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.php
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 1 test using 1 worker
 [1/1] [automatewoo-local] › automatewoo/local/example.spec.js:9:5 › I can see my plugin menu
 Slow test file: [automatewoo-local] › automatewoo/local/example.spec.js (<time>s)

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_run_with_snapshot__1.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.php
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 2 tests using 1 worker
 [1/2] [deli-local] › deli/local/activate-theme.spec.js:4:5 › I can activate Deli
 [deli-local] › deli/local/activate-theme.spec.js:4:5 › I can activate Deli

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_run_with_snapshot__2.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.php
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 2 tests using 1 worker
 [1/2] [deli-local] › deli/local/activate-theme.spec.js:4:5 › I can activate Deli
 [2/2] [deli-local] › deli/local/example.spec.js:9:5 › I can see my plugin menu

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_runs_scaffolded_e2e__1.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.php
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 1 test using 1 worker
 [1/1] [automatewoo-local] › automatewoo/local/example.spec.js:9:5 › I can see my plugin menu
 1 passed (TIME)

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_tag_and_run_test__1.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-tag-and-run/bootstrap/bootstrap.php
 Bootstrapping automatewoo /qit/tests/e2e/automatewoo/self-test-tag-and-run/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 1 test using 1 worker
 [1/1] [automatewoo-self-test-tag-and-run] › automatewoo/self-test-tag-and-run/example.spec.js:9:5 › I can see my plugin menu
 1 passed (TIME)

_tests/custom_tests/tests/__snapshots__/RunE2ETest__test_theme_as_sut__1.txt

@@ -8,6 +8,7 @@ Bootstrapping Plugins
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.php
 Bootstrapping deli /qit/tests/e2e/deli/local/bootstrap/bootstrap.sh
 Running E2E Tests
+Test artifacts being saved to: /tmp-normalized/qit-results-normalizedid
 Running 2 tests using 1 worker
 [1/2] [deli-local] › deli/local/activate-theme.spec.js:4:5 › I can activate Deli
 [2/2] [deli-local] › deli/local/example.spec.js:9:5 › I can see my plugin menu

_tests/managed_tests/QITSelfTests.php

@@ -115,7 +115,11 @@ class Context {
 
 	run_test_runs( generate_test_runs( $test_types ) );
 } catch ( \Exception $e ) {
-	$GLOBALS['parallelOutput']->addRawOutput( $e->getMessage() );
+	if ( isset( $GLOBALS['parallelOutput'] ) && $GLOBALS['parallelOutput'] instanceof ParallelOutput ) {
+		$GLOBALS['parallelOutput']->addRawOutput( $e->getMessage() );
+	} else {
+		echo $e->getMessage();
+	}
 	die( 1 );
 }
 

src/src/Commands/CustomTests/RunE2ECommand.php

@@ -134,6 +134,13 @@ protected function execute( InputInterface $input, OutputInterface $output ): in
 		$update_snapshots = $input->getOption( 'update_snapshots' );
 		$pw_options       = $input->getOption( 'pw_options' ) ?? '';
 
+		if ( ! empty( $pw_options ) ) {
+			// Remove wrapping double quotes if they exist.
+			if ( substr( $pw_options, 0, 1 ) === '"' && substr( $pw_options, - 1 ) === '"' ) {
+				$pw_options = substr( $pw_options, 1, - 1 );
+			}
+		}
+
 		if ( ! empty( $update_snapshots ) ) {
 			$pw_options .= ' --update-snapshots';
 		}

src/src/Environment/EnvironmentVersionResolver.php

@@ -17,7 +17,10 @@ public static function resolve_woo( string $woo ) {
 		} elseif ( $woo === 'rc' ) {
 			throw new \InvalidArgumentException( 'Please specify a RC version, such as "1.2.3-rc.1", or use "nightly".' );
 		} elseif ( $woo === 'stable' ) {
-			$woo = 'woocommerce';
+			$woo = [
+				'slug'   => 'woocommerce',
+				'source' => 'https://downloads.wordpress.org/plugin/woocommerce.latest-stable.zip',
+			];
 		} else {
 			$woo = "https://github.com/woocommerce/woocommerce/releases/download/$woo/woocommerce.zip";
 		}

src/src/Environment/ExtensionDownload/Handlers/Handler.php

@@ -46,15 +46,26 @@ abstract public function maybe_download_extensions( array $extensions, string $c
 	 * @param string $type The type of the extension to make a path for.
 	 * @param string $extension_identifier The extension identifier.
 	 * @param string $extension_version The extension version.
+	 * @param string $extension_source The source of the extension, eg: a URL, etc.
 	 * @param string $cache_burst A cache burst string, defaults to the week of the year.
 	 * @param string $file_format The file format.
 	 *
 	 * @return string The cache path.
 	 */
-	protected function make_cache_path( string $cache_dir, string $type, string $extension_identifier, string $extension_version, string $cache_burst = '', string $file_format = 'zip' ): string {
+	protected function make_cache_path( string $cache_dir, string $type, string $extension_identifier, string $extension_version, string $extension_source, string $cache_burst = '', string $file_format = 'zip' ): string {
 		if ( empty( $cache_burst ) ) {
-			// Get the number of the day, from 1 to 365 - basically means the cache is busted every day or so.
-			$cache_burst = gmdate( 'z' );
+			// If we have a version, cache it for one day.
+			if ( $extension_version !== 'undefined' ) {
+				// Get the number of the day, from 1 to 365 - basically means the cache is busted every day or so.
+				$cache_burst = gmdate( 'z' );
+			} else {
+				/*
+				 * Otherwise, cache it for 1 minute.
+				 * This is because we don't know what version we should be fetching, so we can't cache burst it with confidence.
+				 * We cache it for 1 minute which is just enough to throttle requests.
+				 */
+				$cache_burst = gmdate( 'YmdHi' );
+			}
 		}
 
 		// Make sure $type is as expected.
@@ -72,7 +83,9 @@ protected function make_cache_path( string $cache_dir, string $type, string $ext
 			throw new \InvalidArgumentException( sprintf( 'Invalid extension identifier "%s", should be a valid plugin slug.', $extension_identifier ) );
 		}
 
-		$cache_path = "$cache_dir/$type/$extension_identifier-$extension_version-$cache_burst.$file_format";
+		$source_hash = md5( $extension_source );
+
+		$cache_path = "$cache_dir/$type/$extension_identifier-$source_hash-$extension_version-$cache_burst.$file_format";
 
 		if ( ! file_exists( dirname( $cache_path ) ) ) {
 			if ( ! mkdir( dirname( $cache_path ), 0755, true ) ) {

src/src/Environment/ExtensionDownload/Handlers/QITHandler.php

@@ -102,7 +102,7 @@ public function maybe_download_extensions( array $extensions, string $cache_dir
 				continue;
 			}
 
-			$cache_file = $this->make_cache_path( $cache_dir, $e->type, $e->slug, $e->version );
+			$cache_file = $this->make_cache_path( $cache_dir, $e->type, $e->slug, $e->version, $e->source );
 
 			// Cache hit?
 			if ( file_exists( $cache_file ) ) {

src/src/Environment/ExtensionDownload/Handlers/URLHandler.php

@@ -37,7 +37,7 @@ public function maybe_download_extensions( array $extensions, string $cache_dir
 			// As version is "undefined", cache burst is shorter: Hour of the day (0-24).
 			$cache_burst = gmdate( 'G' );
 
-			$cache_file = $this->make_cache_path( $cache_dir, $e->type, $e->slug, $e->version, $cache_burst );
+			$cache_file = $this->make_cache_path( $cache_dir, $e->type, $e->slug, $e->version, $e->source, $cache_burst );
 
 			// Cache hit?
 			if ( file_exists( $cache_file ) ) {

src/src/LocalTests/E2E/Runner/PlaywrightRunner.php

@@ -54,6 +54,8 @@ public function run_test( E2EEnvInfo $env_info, array $test_infos, TestResult $t
 			}
 		}
 
+		$this->output->writeln( sprintf( 'Test artifacts being saved to: %s', $results_dir ) );
+
 		// Generate playwright-config.
 		$process = new Process( [ PHP_BINARY, $env_info->temporary_env . '/playwright/playwright-config-generator.php' ] );
 		$process->setEnv( [