Refactor integration tests to make requests to wordpress.org

.buildkite/integration-tests.yml

@@ -0,0 +1,16 @@
+steps:
+  - label: ":wordpress: :rust: WordPress.org API"
+    command: |
+      echo "--- :wordpress: Preparing"
+      ./wordpress_org_api_integration_tests/replace-test-parameters.sh
+      echo "--- :rust: Testing"
+      make test-rust-integration-wordpress-org-api
+    agents:
+      queue: tumblr-metal
+
+notify:
+  - slack:
+      channels:
+        - "#wordpress-rs"
+      message: "wordpress-org-api-integration-tests failed."
+    if: build.state == "failed"

.buildkite/pipeline.yml

@@ -201,11 +201,6 @@ steps:
           WORDPRESS_VERSION: "{{matrix}}"
         matrix: *wordpress_version_matrix
 
-      - label: ":wordpress: :rust: WordPress.org API"
-        command: |
-          echo "--- :rust: Testing"
-          make test-rust-integration-wordpress-org-api
-
       - label: ":wordpress: :kotlin: WordPress {{matrix}}"
         command: ".buildkite/commands/run-kotlin-integration-tests.sh"
         env:

Makefile

@@ -204,7 +204,6 @@ test-rust-integration:
 	docker exec -i wordpress /bin/bash < ./scripts/run-rust-integration-tests.sh
 
 test-rust-integration-wordpress-org-api:
-	@test -d target/wordpress-org-plugin-directory || ./scripts/plugin-directory.sh download_from_s3
 	$(rust_docker_run) cargo test --package wordpress_org_api_integration_tests
 
 test-kotlin-integration:

wordpress_org_api/src/plugin_directory.rs

@@ -15,6 +15,7 @@ pub struct PluginInformation {
     #[serde(deserialize_with = "deserialize_default_values")]
     pub author_profile: String,
     #[serde(deserialize_with = "deserialize_default_values")]
+    #[serde(default)]
     pub contributors: HashMap<String, ContributorDetails>,
     #[serde(deserialize_with = "deserialize_default_values")]
     pub requires: String,
@@ -26,30 +27,40 @@ pub struct PluginInformation {
     pub rating: u32,
     pub ratings: Ratings,
     pub num_ratings: u32,
+    #[serde(default)]
     pub support_url: String,
     pub support_threads: u32,
     pub support_threads_resolved: u32,
     pub active_installs: u64,
     pub last_updated: String,
     pub added: String,
     pub homepage: String,
+    #[serde(default)]
     pub sections: HashMap<String, String>,
     pub download_link: String,
     #[serde(deserialize_with = "deserialize_default_values")]
+    #[serde(default)]
     pub upgrade_notice: HashMap<String, String>,
+    #[serde(default)]
     pub screenshots: Screenshots,
     #[serde(deserialize_with = "deserialize_default_values")]
     pub tags: HashMap<String, String>,
     #[serde(deserialize_with = "deserialize_default_values")]
+    #[serde(default)]
     pub versions: HashMap<String, String>,
     #[serde(deserialize_with = "deserialize_default_values")]
+    #[serde(default)]
     pub business_model: String,
+    #[serde(default)]
     pub repository_url: String,
+    #[serde(default)]
     pub commercial_support_url: String,
     pub donate_link: String,
     #[serde(deserialize_with = "deserialize_default_values")]
+    #[serde(default)]
     pub banners: Banners,
     pub icons: Option<Icons>,
+    #[serde(default)]
     pub preview_link: String,
 }
 
@@ -82,6 +93,12 @@ pub enum Screenshots {
     List(Vec<Screenshot>),
 }
 
+impl Default for Screenshots {
+    fn default() -> Self {
+        Screenshots::List(vec![])
+    }
+}
+
 #[derive(Deserialize, Debug)]
 pub struct Screenshot {
     pub src: String,
@@ -105,3 +122,16 @@ pub struct Icons {
     pub svg: Option<String>,
     pub default: Option<String>,
 }
+
+#[derive(Deserialize, Debug)]
+pub struct QueryPluginResponse {
+    pub info: QueryPluginResponseInfo,
+    pub plugins: Vec<PluginInformation>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct QueryPluginResponseInfo {
+    pub page: i64,
+    pub pages: i64,
+    pub results: i64,
+}

wordpress_org_api_integration_tests/Cargo.toml

@@ -8,5 +8,9 @@ publish = false
 wordpress_org_api = { path = "../wordpress_org_api" }
 
 [dev-dependencies]
+reqwest = { workspace = true, features = [ "json" ] }
 rstest = { workspace = true }
+rstest_reuse = { workspace = true }
+serde = { workspace = true, features = [ "derive" ] }
 serde_json = { workspace = true }
+tokio = { workspace = true, features = [ "full" ] }

wordpress_org_api_integration_tests/replace-test-parameters.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -euo pipefail
+
+save_plugin_slug() {
+    url=$1
+    curl -s "$url" | jq -r '.plugins[].slug' | xargs -I {} touch "$SLUG_DIR/{}"
+}
+
+OUTPUT_FILE="wordpress_org_api_integration_tests/tests/plugin_directory_parameters.rs"
+
+echo "Prepare integration tests parameters"
+
+API_URL="https://api.wordpress.org/plugins/info/1.2/"
+PAGE_SIZE=200
+PARALLEL_JOBS=20
+
+current_page=1
+response=$(curl -s "$API_URL?action=query_plugins&request%5Bpage%5D=$current_page&request%5Bper_page%5D=$PAGE_SIZE")
+total_pages=$(echo "$response" | jq -r '.info.pages')
+if [[ -z "$total_pages" || "$total_pages" -eq 0 ]]; then
+    echo "Failed to fetch pagination info or no pages available."
+    exit 1
+fi
+echo "Total pages to process: $total_pages"
+
+query_plugins_api_urls=()
+current_page=1
+while [[ $current_page -le $total_pages ]]; do
+    query_plugins_api_urls+=("$API_URL?action=query_plugins&request%5Bpage%5D=$current_page&request%5Bper_page%5D=$PAGE_SIZE")
+    current_page=$((current_page + 1))
+done
+
+export -f save_plugin_slug
+export API_URL
+export SLUG_DIR="target/wordpress-org-plugin-directory/slugs"
+echo "Saving plugin slugs to $SLUG_DIR"
+rm -rf "$SLUG_DIR" && mkdir -p "$SLUG_DIR"
+echo "${query_plugins_api_urls[@]}" | xargs -n 1 -P "$PARALLEL_JOBS" bash -c 'save_plugin_slug "$@"' _
+
+slugs=()
+while IFS='' read -r line; do slugs+=("$line"); done < <(ls -1 "$SLUG_DIR")
+
+{
+    echo '#![allow(unused_macros)]'
+    echo ''
+    echo '#[rstest_reuse::template]'
+    echo '#[rstest::rstest]'
+    for url in "${query_plugins_api_urls[@]}"; do
+        echo "#[case(\"$url\")]"
+    done
+    echo 'pub fn query_plugins_api_url(#[case] url: &str) {}'
+    echo ''
+
+    midpoint=$((${#slugs[@]} / 2))
+    first_half=("${slugs[@]:0:midpoint}")
+    second_half=("${slugs[@]:midpoint}")
+
+    echo '#[rstest_reuse::template]'
+    echo '#[rstest::rstest]'
+    for slug in "${first_half[@]}"; do
+        echo "#[case(\"$slug\")]"
+    done
+    echo 'pub fn plugin_information_slug_1(#[case] slug: &str) {}'
+    echo ''
+
+    echo '#[rstest_reuse::template]'
+    echo '#[rstest::rstest]'
+    for slug in "${second_half[@]}"; do
+        echo "#[case(\"$slug\")]"
+    done
+    echo 'pub fn plugin_information_slug_2(#[case] slug: &str) {}'
+    echo ''
+} > "$OUTPUT_FILE"

wordpress_org_api_integration_tests/tests/plugin_directory_parameters.rs

@@ -0,0 +1,26 @@
+#![allow(unused_macros)]
+
+// Run replace-test-parameters.sh to replace the parameters defined in this file
+// with the full plugin directory on wordpress.org.
+//
+// Please note: build time will increase significantly after running the script.
+//
+// IMPORTANT: DO NOT COMMIT THE CHANGES TO THIS FILE AFTER RUNNING THE SCRIPT.
+
+#[rstest_reuse::template]
+#[rstest::rstest]
+#[case("https://api.wordpress.org/plugins/info/1.2/?action=query_plugins&request%5Bpage%5D=1&request%5Bper_page%5D=200")]
+#[case("https://api.wordpress.org/plugins/info/1.2/?action=query_plugins&request%5Bpage%5D=2&request%5Bper_page%5D=200")]
+#[case("https://api.wordpress.org/plugins/info/1.2/?action=query_plugins&request%5Bpage%5D=273&request%5Bper_page%5D=200")]
+#[case("https://api.wordpress.org/plugins/info/1.2/?action=query_plugins&request%5Bpage%5D=274&request%5Bper_page%5D=200")]
+pub fn query_plugins_api_url(#[case] url: &str) {}
+
+#[rstest_reuse::template]
+#[rstest::rstest]
+#[case("jetpack")]
+pub fn plugin_information_slug_1(#[case] slug: &str) {}
+
+#[rstest_reuse::template]
+#[rstest::rstest]
+#[case("woocommerce")]
+pub fn plugin_information_slug_2(#[case] slug: &str) {}